Code Repositories xandikos / 41b85e3
Implement a VdirStore. Jelmer Vernooń≥ 1 year, 3 months ago
2 changed file(s) with 379 addition(s) and 0 deletion(s). Raw diff Collapse all Expand all
0 # Xandikos
1 # Copyright (C) 2016-2017 Jelmer Vernooń≥ <jelmer@jelmer.uk>, et al.
2 #
3 # This program is free software; you can redistribute it and/or
4 # modify it under the terms of the GNU General Public License
5 # as published by the Free Software Foundation; version 3
6 # of the License or (at your option) any later version of
7 # the License.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software
16 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
17 # MA 02110-1301, USA.
18
19 """vdir store.
20
21 See https://github.com/pimutils/vdirsyncer/blob/master/docs/vdir.rst
22 """
23
24 import errno
25 import logging
26 import os
27 import shutil
28 import uuid
29
30 from . import (
31 MIMETYPES,
32 Store,
33 DuplicateUidError,
34 InvalidETag,
35 InvalidFileContents,
36 NoSuchItem,
37 open_by_content_type,
38 open_by_extension,
39 )
40 from .config import CollectionConfig
41
42
43 DEFAULT_ENCODING = 'utf-8'
44
45
46 logger = logging.getLogger(__name__)
47
48
49 class VdirStore(Store):
50 """A Store backed by a Vdir directory.
51 """
52
53 def __init__(self, path, check_for_duplicate_uids=True):
54 super(VdirStore, self).__init__()
55 self.path = path
56 self._check_for_duplicate_uids = check_for_duplicate_uids
57 # Set of blob ids that have already been scanned
58 self._fname_to_uid = {}
59 # Maps uids to (sha, fname)
60 self._uid_to_fname = {}
61
62 @property
63 def config(self):
64 return CollectionConfig()
65
66 def __repr__(self):
67 return "%s(%r)" % (type(self).__name__, self.path)
68
69 def _get_etag(self, name):
70 path = os.path.join(self.path, name)
71 try:
72 st = os.stat(path)
73 except IOError as e:
74 if e.errno == errno.ENOENT:
75 raise KeyError
76 raise
77 mtime = getattr(st, 'st_mtime_ns', None)
78 if mtime is None:
79 mtime = st.st_mtime
80 return '{:.9f}'.format(mtime)
81
82 def _get_raw(self, name, etag=None):
83 """Get the raw contents of an object.
84
85 :param name: Name of the item
86 :param etag: Optional etag
87 :return: raw contents as chunks
88 """
89 if etag is None:
90 etag = self._get_etag(name)
91 path = os.path.join(self.path, name)
92 try:
93 with open(path, 'rb') as f:
94 return [f.read()]
95 except IOError as e:
96 if e.errno == errno.ENOENT:
97 raise KeyError
98 raise
99
100 def _scan_uids(self):
101 removed = set(self._fname_to_uid.keys())
102 for (name, content_type, etag) in self.iter_with_etag():
103 if name in removed:
104 removed.remove(name)
105 if (name in self._fname_to_uid and
106 self._fname_to_uid[name][0] == etag):
107 continue
108 fi = open_by_extension(self._get_raw(name, etag), name,
109 self.extra_file_handlers)
110 try:
111 uid = fi.get_uid()
112 except KeyError:
113 logger.warning('No UID found in file %s', name)
114 uid = None
115 except InvalidFileContents:
116 logging.warning('Unable to parse file %s', name)
117 uid = None
118 except NotImplementedError:
119 # This file type doesn't support UIDs
120 uid = None
121 self._fname_to_uid[name] = (etag, uid)
122 if uid is not None:
123 self._uid_to_fname[uid] = (name, etag)
124 for name in removed:
125 (unused_etag, uid) = self._fname_to_uid[name]
126 if uid is not None:
127 del self._uid_to_fname[uid]
128 del self._fname_to_uid[name]
129
130 def _check_duplicate(self, uid, name, replace_etag):
131 if uid is not None and self._check_for_duplicate_uids:
132 self._scan_uids()
133 try:
134 (existing_name, _) = self._uid_to_fname[uid]
135 except KeyError:
136 pass
137 else:
138 if existing_name != name:
139 raise DuplicateUidError(uid, existing_name, name)
140
141 try:
142 etag = self._get_etag(name)
143 except KeyError:
144 etag = None
145 if replace_etag is not None and etag != replace_etag:
146 raise InvalidETag(name, etag, replace_etag)
147 return etag
148
149 def import_one(self, name, content_type, data, message=None, author=None,
150 replace_etag=None):
151 """Import a single object.
152
153 :param name: name of the object
154 :param content_type: Content type
155 :param data: serialized object as list of bytes
156 :param message: Commit message
157 :param author: Optional author
158 :param replace_etag: optional etag of object to replace
159 :raise InvalidETag: when the name already exists but with different
160 etag
161 :raise DuplicateUidError: when the uid already exists
162 :return: etag
163 """
164 if content_type is None:
165 fi = open_by_extension(data, name, self.extra_file_handlers)
166 else:
167 fi = open_by_content_type(
168 data, content_type, self.extra_file_handlers)
169 if name is None:
170 name = str(uuid.uuid4())
171 extension = MIMETYPES.guess_extension(content_type)
172 if extension is not None:
173 name += extension
174 fi.validate()
175 try:
176 uid = fi.get_uid()
177 except (KeyError, NotImplementedError):
178 uid = None
179 self._check_duplicate(uid, name, replace_etag)
180
181 # TODO(jelmer): Check that extensions match content type:
182 # if this is a vCard, the extension should be .vcf
183 # if this is a iCalendar, the extension should be .ics
184 # TODO(jelmer): check that a UID is present and that all UIDs are the
185 # same
186 path = os.path.join(self.path, name)
187 tmppath = os.path.join(self.path, name + '.tmp')
188 with open(tmppath, 'wb') as f:
189 for chunk in data:
190 f.write(chunk)
191 os.rename(tmppath, path)
192 return (name, self._get_etag(name))
193
194 def iter_with_etag(self, ctag=None):
195 """Iterate over all items in the store with etag.
196
197 :param ctag: Ctag to iterate for
198 :yield: (name, content_type, etag) tuples
199 """
200 for name in os.listdir(self.path):
201 if name.endswith('.tmp'):
202 continue
203 if name.endswith('.ics'):
204 content_type = 'text/calendar'
205 elif name.endswith('.vcf'):
206 content_type = 'text/vcard'
207 else:
208 continue
209 yield (name, content_type, self._get_etag(name))
210
211 @classmethod
212 def create(cls, path):
213 """Create a new store backed by a Vdir on disk.
214
215 :return: A `VdirStore`
216 """
217 os.mkdir(path)
218 return cls(path)
219
220 @classmethod
221 def open_from_path(cls, path):
222 """Open a VdirStore from a path.
223
224 :param path: Path
225 :return: A `VdirStore`
226 """
227 return cls(path)
228
229 def get_description(self):
230 """Get extended description.
231
232 :return: repository description as string
233 """
234 raise NotImplementedError(self.get_description)
235
236 def set_description(self, description):
237 """Set extended description.
238
239 :param description: repository description as string
240 """
241 raise NotImplementedError(self.set_description)
242
243 def set_comment(self, comment):
244 """Set comment.
245
246 :param comment: Comment
247 """
248 raise NotImplementedError(self.set_comment)
249
250 def get_comment(self):
251 """Get comment.
252
253 :return: Comment
254 """
255 raise NotImplementedError(self.get_comment)
256
257 def _read_metadata(self, name):
258 try:
259 with open(os.path.join(self.path, name), 'r') as f:
260 return f.read().strip()
261 except EnvironmentError:
262 return None
263
264 def _write_metadata(self, name, data):
265 path = os.path.join(self.path, name)
266 if data is not None:
267 with open(path, 'w') as f:
268 f.write(data)
269 else:
270 os.unlink(path)
271
272 def get_color(self):
273 """Get color.
274
275 :return: A Color code, or None
276 """
277 color = self._read_metadata('color')
278 assert color.startswith('#')
279 return color
280
281 def set_color(self, color):
282 """Set the color code for this store."""
283 assert color.startswith('#')
284 self._write_metadata('color', color)
285
286 def get_displayname(self):
287 """Get display name.
288
289 :return: The display name, or None if not set
290 """
291 return self._read_metadata('displayname')
292
293 def set_displayname(self, displayname):
294 """Set the display name.
295
296 :param displayname: New display name
297 """
298 self._write_metadata('displayname', displayname)
299
300 def set_type(self, store_type):
301 """Set store type.
302
303 :param store_type: New store type (one of VALID_STORE_TYPES)
304 """
305 raise NotImplementedError(self.set_type)
306
307 def get_type(self):
308 """Get store type.
309 """
310 raise NotImplementedError(self.get_type)
311
312 def iter_changes(self, old_ctag, new_ctag):
313 """Get changes between two versions of this store.
314
315 :param old_ctag: Old ctag (None for empty Store)
316 :param new_ctag: New ctag
317 :return: Iterator over (name, content_type, old_etag, new_etag)
318 """
319 raise NotImplementedError(self.iter_changes)
320
321 def destroy(self):
322 """Destroy this store."""
323 shutil.rmtree(self.path)
324
325 def delete_one(self, name, message=None, author=None, etag=None):
326 """Delete an item.
327
328 :param name: Filename to delete
329 :param message: Commit message
330 :param author: Optional author
331 :param etag: Optional mandatory etag of object to remove
332 :raise NoSuchItem: when the item doesn't exist
333 :raise InvalidETag: If the specified ETag doesn't match the curren
334 """
335 path = os.path.join(self.path, name)
336 if etag is not None:
337 try:
338 current_etag = self._get_etag(name)
339 except KeyError:
340 raise NoSuchItem(name)
341 if etag != current_etag:
342 raise InvalidETag(name, etag, current_etag)
343 try:
344 os.unlink(path)
345 except EnvironmentError as e:
346 if e.errno == errno.ENOENT:
347 raise NoSuchItem(path)
348 raise
349
350 def get_ctag(self):
351 """Return the ctag for this store."""
352 raise NotImplementedError(self.get_ctag)
353
354 def subdirectories(self):
355 """Returns subdirectories to probe for other stores.
356
357 :return: List of names
358 """
359 ret = []
360 for name in os.listdir(self.path):
361 p = os.path.join(self.path, name)
362 if os.path.isdir(p):
363 ret.append(name)
364 return ret
3131 DuplicateUidError, File, InvalidETag, NoSuchItem)
3232 from xandikos.store.git import (
3333 GitStore, BareGitStore, TreeGitStore)
34 from xandikos.store.vdir import (
35 VdirStore)
3436
3537 EXAMPLE_VCALENDAR1 = b"""\
3638 BEGIN:VCALENDAR
183185 set(gc.iter_with_etag()))
184186
185187
188 class VdirStoreTest(BaseStoreTest, unittest.TestCase):
189
190 kls = VdirStore
191
192 def create_store(self):
193 d = tempfile.mkdtemp()
194 self.addCleanup(shutil.rmtree, d)
195 store = self.kls.create(os.path.join(d, 'store'))
196 store.load_extra_file_handler(ICalendarFile)
197 return store
198
199
186200 class BaseGitStoreTest(BaseStoreTest):
187201
188202 kls = None