Example File Systems

pyfuse3 comes with several example file systems in the examples directory of the release tarball. For completeness, these examples are also included here.

Single-file, Read-only File System

(shipped as examples/lltest.py)

  1#!/usr/bin/env python3
  2# -*- coding: utf-8 -*-
  3'''
  4hello.py - Example file system for pyfuse3.
  5
  6This program presents a static file system containing a single file.
  7
  8Copyright © 2015 Nikolaus Rath <Nikolaus.org>
  9Copyright © 2015 Gerion Entrup.
 10
 11Permission is hereby granted, free of charge, to any person obtaining a copy of
 12this software and associated documentation files (the "Software"), to deal in
 13the Software without restriction, including without limitation the rights to
 14use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
 15the Software, and to permit persons to whom the Software is furnished to do so.
 16
 17THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 18IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 19FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 20COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 21IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 22CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 23'''
 24
 25import os
 26import sys
 27
 28# If we are running from the pyfuse3 source directory, try
 29# to load the module from there first.
 30basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..'))
 31if (os.path.exists(os.path.join(basedir, 'setup.py')) and
 32    os.path.exists(os.path.join(basedir, 'src', 'pyfuse3.pyx'))):
 33    sys.path.insert(0, os.path.join(basedir, 'src'))
 34
 35from argparse import ArgumentParser
 36import stat
 37import logging
 38import errno
 39import pyfuse3
 40import trio
 41
 42try:
 43    import faulthandler
 44except ImportError:
 45    pass
 46else:
 47    faulthandler.enable()
 48
 49log = logging.getLogger(__name__)
 50
 51class TestFs(pyfuse3.Operations):
 52    def __init__(self):
 53        super(TestFs, self).__init__()
 54        self.hello_name = b"message"
 55        self.hello_inode = pyfuse3.ROOT_INODE+1
 56        self.hello_data = b"hello world\n"
 57
 58    async def getattr(self, inode, ctx=None):
 59        entry = pyfuse3.EntryAttributes()
 60        if inode == pyfuse3.ROOT_INODE:
 61            entry.st_mode = (stat.S_IFDIR | 0o755)
 62            entry.st_size = 0
 63        elif inode == self.hello_inode:
 64            entry.st_mode = (stat.S_IFREG | 0o644)
 65            entry.st_size = len(self.hello_data)
 66        else:
 67            raise pyfuse3.FUSEError(errno.ENOENT)
 68
 69        stamp = int(1438467123.985654 * 1e9)
 70        entry.st_atime_ns = stamp
 71        entry.st_ctime_ns = stamp
 72        entry.st_mtime_ns = stamp
 73        entry.st_gid = os.getgid()
 74        entry.st_uid = os.getuid()
 75        entry.st_ino = inode
 76
 77        return entry
 78
 79    async def lookup(self, parent_inode, name, ctx=None):
 80        if parent_inode != pyfuse3.ROOT_INODE or name != self.hello_name:
 81            raise pyfuse3.FUSEError(errno.ENOENT)
 82        return self.getattr(self.hello_inode)
 83
 84    async def opendir(self, inode, ctx):
 85        if inode != pyfuse3.ROOT_INODE:
 86            raise pyfuse3.FUSEError(errno.ENOENT)
 87        return inode
 88
 89    async def readdir(self, fh, start_id, token):
 90        assert fh == pyfuse3.ROOT_INODE
 91
 92        # only one entry
 93        if start_id == 0:
 94            pyfuse3.readdir_reply(
 95                token, self.hello_name, await self.getattr(self.hello_inode), 1)
 96        return
 97
 98    async def open(self, inode, flags, ctx):
 99        if inode != self.hello_inode:
100            raise pyfuse3.FUSEError(errno.ENOENT)
101        if flags & os.O_RDWR or flags & os.O_WRONLY:
102            raise pyfuse3.FUSEError(errno.EACCES)
103        return pyfuse3.FileInfo(fh=inode)
104
105    async def read(self, fh, off, size):
106        assert fh == self.hello_inode
107        return self.hello_data[off:off+size]
108
109def init_logging(debug=False):
110    formatter = logging.Formatter('%(asctime)s.%(msecs)03d %(threadName)s: '
111                                  '[%(name)s] %(message)s', datefmt="%Y-%m-%d %H:%M:%S")
112    handler = logging.StreamHandler()
113    handler.setFormatter(formatter)
114    root_logger = logging.getLogger()
115    if debug:
116        handler.setLevel(logging.DEBUG)
117        root_logger.setLevel(logging.DEBUG)
118    else:
119        handler.setLevel(logging.INFO)
120        root_logger.setLevel(logging.INFO)
121    root_logger.addHandler(handler)
122
123def parse_args():
124    '''Parse command line'''
125
126    parser = ArgumentParser()
127
128    parser.add_argument('mountpoint', type=str,
129                        help='Where to mount the file system')
130    parser.add_argument('--debug', action='store_true', default=False,
131                        help='Enable debugging output')
132    parser.add_argument('--debug-fuse', action='store_true', default=False,
133                        help='Enable FUSE debugging output')
134    return parser.parse_args()
135
136
137def main():
138    options = parse_args()
139    init_logging(options.debug)
140
141    testfs = TestFs()
142    fuse_options = set(pyfuse3.default_options)
143    fuse_options.add('fsname=hello')
144    if options.debug_fuse:
145        fuse_options.add('debug')
146    pyfuse3.init(testfs, options.mountpoint, fuse_options)
147    try:
148        trio.run(pyfuse3.main)
149    except:
150        pyfuse3.close(unmount=False)
151        raise
152
153    pyfuse3.close()
154
155
156if __name__ == '__main__':
157    main()

In-memory File System

(shipped as examples/tmpfs.py)

  1#!/usr/bin/env python3
  2# -*- coding: utf-8 -*-
  3'''
  4tmpfs.py - Example file system for pyfuse3.
  5
  6This file system stores all data in memory.
  7
  8Copyright © 2013 Nikolaus Rath <Nikolaus.org>
  9
 10Permission is hereby granted, free of charge, to any person obtaining a copy of
 11this software and associated documentation files (the "Software"), to deal in
 12the Software without restriction, including without limitation the rights to
 13use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
 14the Software, and to permit persons to whom the Software is furnished to do so.
 15
 16THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 17IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 18FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 19COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 20IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 21CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 22'''
 23
 24import os
 25import sys
 26
 27# If we are running from the pyfuse3 source directory, try
 28# to load the module from there first.
 29basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..'))
 30if (os.path.exists(os.path.join(basedir, 'setup.py')) and
 31    os.path.exists(os.path.join(basedir, 'src', 'pyfuse3.pyx'))):
 32    sys.path.insert(0, os.path.join(basedir, 'src'))
 33
 34import pyfuse3
 35import errno
 36import stat
 37from time import time
 38import sqlite3
 39import logging
 40from collections import defaultdict
 41from pyfuse3 import FUSEError
 42from argparse import ArgumentParser
 43import trio
 44
 45try:
 46    import faulthandler
 47except ImportError:
 48    pass
 49else:
 50    faulthandler.enable()
 51
 52log = logging.getLogger()
 53
 54class Operations(pyfuse3.Operations):
 55    '''An example filesystem that stores all data in memory
 56
 57    This is a very simple implementation with terrible performance.
 58    Don't try to store significant amounts of data. Also, there are
 59    some other flaws that have not been fixed to keep the code easier
 60    to understand:
 61
 62    * atime, mtime and ctime are not updated
 63    * generation numbers are not supported
 64    * lookup counts are not maintained
 65    '''
 66
 67    enable_writeback_cache = True
 68
 69    def __init__(self):
 70        super(Operations, self).__init__()
 71        self.db = sqlite3.connect(':memory:')
 72        self.db.text_factory = str
 73        self.db.row_factory = sqlite3.Row
 74        self.cursor = self.db.cursor()
 75        self.inode_open_count = defaultdict(int)
 76        self.init_tables()
 77
 78    def init_tables(self):
 79        '''Initialize file system tables'''
 80
 81        self.cursor.execute("""
 82        CREATE TABLE inodes (
 83            id        INTEGER PRIMARY KEY,
 84            uid       INT NOT NULL,
 85            gid       INT NOT NULL,
 86            mode      INT NOT NULL,
 87            mtime_ns  INT NOT NULL,
 88            atime_ns  INT NOT NULL,
 89            ctime_ns  INT NOT NULL,
 90            target    BLOB(256) ,
 91            size      INT NOT NULL DEFAULT 0,
 92            rdev      INT NOT NULL DEFAULT 0,
 93            data      BLOB
 94        )
 95        """)
 96
 97        self.cursor.execute("""
 98        CREATE TABLE contents (
 99            rowid     INTEGER PRIMARY KEY AUTOINCREMENT,
100            name      BLOB(256) NOT NULL,
101            inode     INT NOT NULL REFERENCES inodes(id),
102            parent_inode INT NOT NULL REFERENCES inodes(id),
103
104            UNIQUE (name, parent_inode)
105        )""")
106
107        # Insert root directory
108        now_ns = int(time() * 1e9)
109        self.cursor.execute("INSERT INTO inodes (id,mode,uid,gid,mtime_ns,atime_ns,ctime_ns) "
110                            "VALUES (?,?,?,?,?,?,?)",
111                            (pyfuse3.ROOT_INODE, stat.S_IFDIR | stat.S_IRUSR | stat.S_IWUSR
112                              | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH
113                              | stat.S_IXOTH, os.getuid(), os.getgid(), now_ns, now_ns, now_ns))
114        self.cursor.execute("INSERT INTO contents (name, parent_inode, inode) VALUES (?,?,?)",
115                            (b'..', pyfuse3.ROOT_INODE, pyfuse3.ROOT_INODE))
116
117
118    def get_row(self, *a, **kw):
119        self.cursor.execute(*a, **kw)
120        try:
121            row = next(self.cursor)
122        except StopIteration:
123            raise NoSuchRowError()
124        try:
125            next(self.cursor)
126        except StopIteration:
127            pass
128        else:
129            raise NoUniqueValueError()
130
131        return row
132
133    async def lookup(self, inode_p, name, ctx=None):
134        if name == '.':
135            inode = inode_p
136        elif name == '..':
137            inode = self.get_row("SELECT * FROM contents WHERE inode=?",
138                                 (inode_p,))['parent_inode']
139        else:
140            try:
141                inode = self.get_row("SELECT * FROM contents WHERE name=? AND parent_inode=?",
142                                     (name, inode_p))['inode']
143            except NoSuchRowError:
144                raise(pyfuse3.FUSEError(errno.ENOENT))
145
146        return await self.getattr(inode, ctx)
147
148
149    async def getattr(self, inode, ctx=None):
150        row = self.get_row('SELECT * FROM inodes WHERE id=?', (inode,))
151
152        entry = pyfuse3.EntryAttributes()
153        entry.st_ino = inode
154        entry.generation = 0
155        entry.entry_timeout = 300
156        entry.attr_timeout = 300
157        entry.st_mode = row['mode']
158        entry.st_nlink = self.get_row("SELECT COUNT(inode) FROM contents WHERE inode=?",
159                                     (inode,))[0]
160        entry.st_uid = row['uid']
161        entry.st_gid = row['gid']
162        entry.st_rdev = row['rdev']
163        entry.st_size = row['size']
164
165        entry.st_blksize = 512
166        entry.st_blocks = 1
167        entry.st_atime_ns = row['atime_ns']
168        entry.st_mtime_ns = row['mtime_ns']
169        entry.st_ctime_ns = row['ctime_ns']
170
171        return entry
172
173    async def readlink(self, inode, ctx):
174        return self.get_row('SELECT * FROM inodes WHERE id=?', (inode,))['target']
175
176    async def opendir(self, inode, ctx):
177        return inode
178
179    async def readdir(self, inode, off, token):
180        if off == 0:
181            off = -1
182
183        cursor2 = self.db.cursor()
184        cursor2.execute("SELECT * FROM contents WHERE parent_inode=? "
185                        'AND rowid > ? ORDER BY rowid', (inode, off))
186
187        for row in cursor2:
188            pyfuse3.readdir_reply(
189                token, row['name'], await self.getattr(row['inode']), row['rowid'])
190
191    async def unlink(self, inode_p, name,ctx):
192        entry = await self.lookup(inode_p, name)
193
194        if stat.S_ISDIR(entry.st_mode):
195            raise pyfuse3.FUSEError(errno.EISDIR)
196
197        self._remove(inode_p, name, entry)
198
199    async def rmdir(self, inode_p, name, ctx):
200        entry = await self.lookup(inode_p, name)
201
202        if not stat.S_ISDIR(entry.st_mode):
203            raise pyfuse3.FUSEError(errno.ENOTDIR)
204
205        self._remove(inode_p, name, entry)
206
207    def _remove(self, inode_p, name, entry):
208        if self.get_row("SELECT COUNT(inode) FROM contents WHERE parent_inode=?",
209                        (entry.st_ino,))[0] > 0:
210            raise pyfuse3.FUSEError(errno.ENOTEMPTY)
211
212        self.cursor.execute("DELETE FROM contents WHERE name=? AND parent_inode=?",
213                        (name, inode_p))
214
215        if entry.st_nlink == 1 and entry.st_ino not in self.inode_open_count:
216            self.cursor.execute("DELETE FROM inodes WHERE id=?", (entry.st_ino,))
217
218    async def symlink(self, inode_p, name, target, ctx):
219        mode = (stat.S_IFLNK | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
220                stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP |
221                stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH)
222        return await self._create(inode_p, name, mode, ctx, target=target)
223
224    async def rename(self, inode_p_old, name_old, inode_p_new, name_new,
225                     flags, ctx):
226        if flags != 0:
227            raise FUSEError(errno.EINVAL)
228
229        entry_old = await self.lookup(inode_p_old, name_old)
230
231        try:
232            entry_new = await self.lookup(inode_p_new, name_new)
233        except pyfuse3.FUSEError as exc:
234            if exc.errno != errno.ENOENT:
235                raise
236            target_exists = False
237        else:
238            target_exists = True
239
240        if target_exists:
241            self._replace(inode_p_old, name_old, inode_p_new, name_new,
242                          entry_old, entry_new)
243        else:
244            self.cursor.execute("UPDATE contents SET name=?, parent_inode=? WHERE name=? "
245                                "AND parent_inode=?", (name_new, inode_p_new,
246                                                       name_old, inode_p_old))
247
248    def _replace(self, inode_p_old, name_old, inode_p_new, name_new,
249                 entry_old, entry_new):
250
251        if self.get_row("SELECT COUNT(inode) FROM contents WHERE parent_inode=?",
252                        (entry_new.st_ino,))[0] > 0:
253            raise pyfuse3.FUSEError(errno.ENOTEMPTY)
254
255        self.cursor.execute("UPDATE contents SET inode=? WHERE name=? AND parent_inode=?",
256                            (entry_old.st_ino, name_new, inode_p_new))
257        self.db.execute('DELETE FROM contents WHERE name=? AND parent_inode=?',
258                        (name_old, inode_p_old))
259
260        if entry_new.st_nlink == 1 and entry_new.st_ino not in self.inode_open_count:
261            self.cursor.execute("DELETE FROM inodes WHERE id=?", (entry_new.st_ino,))
262
263
264    async def link(self, inode, new_inode_p, new_name, ctx):
265        entry_p = await self.getattr(new_inode_p)
266        if entry_p.st_nlink == 0:
267            log.warn('Attempted to create entry %s with unlinked parent %d',
268                     new_name, new_inode_p)
269            raise FUSEError(errno.EINVAL)
270
271        self.cursor.execute("INSERT INTO contents (name, inode, parent_inode) VALUES(?,?,?)",
272                            (new_name, inode, new_inode_p))
273
274        return await self.getattr(inode)
275
276    async def setattr(self, inode, attr, fields, fh, ctx):
277
278        if fields.update_size:
279            data = self.get_row('SELECT data FROM inodes WHERE id=?', (inode,))[0]
280            if data is None:
281                data = b''
282            if len(data) < attr.st_size:
283                data = data + b'\0' * (attr.st_size - len(data))
284            else:
285                data = data[:attr.st_size]
286            self.cursor.execute('UPDATE inodes SET data=?, size=? WHERE id=?',
287                                (memoryview(data), attr.st_size, inode))
288        if fields.update_mode:
289            self.cursor.execute('UPDATE inodes SET mode=? WHERE id=?',
290                                (attr.st_mode, inode))
291
292        if fields.update_uid:
293            self.cursor.execute('UPDATE inodes SET uid=? WHERE id=?',
294                                (attr.st_uid, inode))
295
296        if fields.update_gid:
297            self.cursor.execute('UPDATE inodes SET gid=? WHERE id=?',
298                                (attr.st_gid, inode))
299
300        if fields.update_atime:
301            self.cursor.execute('UPDATE inodes SET atime_ns=? WHERE id=?',
302                                (attr.st_atime_ns, inode))
303
304        if fields.update_mtime:
305            self.cursor.execute('UPDATE inodes SET mtime_ns=? WHERE id=?',
306                                (attr.st_mtime_ns, inode))
307
308        if fields.update_ctime:
309            self.cursor.execute('UPDATE inodes SET ctime_ns=? WHERE id=?',
310                                (attr.st_ctime_ns, inode))
311        else:
312            self.cursor.execute('UPDATE inodes SET ctime_ns=? WHERE id=?',
313                                (int(time()*1e9), inode))
314
315        return await self.getattr(inode)
316
317    async def mknod(self, inode_p, name, mode, rdev, ctx):
318        return await self._create(inode_p, name, mode, ctx, rdev=rdev)
319
320    async def mkdir(self, inode_p, name, mode, ctx):
321        return await self._create(inode_p, name, mode, ctx)
322
323    async def statfs(self, ctx):
324        stat_ = pyfuse3.StatvfsData()
325
326        stat_.f_bsize = 512
327        stat_.f_frsize = 512
328
329        size = self.get_row('SELECT SUM(size) FROM inodes')[0]
330        stat_.f_blocks = size // stat_.f_frsize
331        stat_.f_bfree = max(size // stat_.f_frsize, 1024)
332        stat_.f_bavail = stat_.f_bfree
333
334        inodes = self.get_row('SELECT COUNT(id) FROM inodes')[0]
335        stat_.f_files = inodes
336        stat_.f_ffree = max(inodes , 100)
337        stat_.f_favail = stat_.f_ffree
338
339        return stat_
340
341    async def open(self, inode, flags, ctx):
342        # Yeah, unused arguments
343        #pylint: disable=W0613
344        self.inode_open_count[inode] += 1
345
346        # Use inodes as a file handles
347        return pyfuse3.FileInfo(fh=inode)
348
349    async def access(self, inode, mode, ctx):
350        # Yeah, could be a function and has unused arguments
351        #pylint: disable=R0201,W0613
352        return True
353
354    async def create(self, inode_parent, name, mode, flags, ctx):
355        #pylint: disable=W0612
356        entry = await self._create(inode_parent, name, mode, ctx)
357        self.inode_open_count[entry.st_ino] += 1
358        return (pyfuse3.FileInfo(fh=entry.st_ino), entry)
359
360    async def _create(self, inode_p, name, mode, ctx, rdev=0, target=None):
361        if (await self.getattr(inode_p)).st_nlink == 0:
362            log.warn('Attempted to create entry %s with unlinked parent %d',
363                     name, inode_p)
364            raise FUSEError(errno.EINVAL)
365
366        now_ns = int(time() * 1e9)
367        self.cursor.execute('INSERT INTO inodes (uid, gid, mode, mtime_ns, atime_ns, '
368                            'ctime_ns, target, rdev) VALUES(?, ?, ?, ?, ?, ?, ?, ?)',
369                            (ctx.uid, ctx.gid, mode, now_ns, now_ns, now_ns, target, rdev))
370
371        inode = self.cursor.lastrowid
372        self.db.execute("INSERT INTO contents(name, inode, parent_inode) VALUES(?,?,?)",
373                        (name, inode, inode_p))
374        return await self.getattr(inode)
375
376    async def read(self, fh, offset, length):
377        data = self.get_row('SELECT data FROM inodes WHERE id=?', (fh,))[0]
378        if data is None:
379            data = b''
380        return data[offset:offset+length]
381
382    async def write(self, fh, offset, buf):
383        data = self.get_row('SELECT data FROM inodes WHERE id=?', (fh,))[0]
384        if data is None:
385            data = b''
386        data = data[:offset] + buf + data[offset+len(buf):]
387
388        self.cursor.execute('UPDATE inodes SET data=?, size=? WHERE id=?',
389                            (memoryview(data), len(data), fh))
390        return len(buf)
391
392    async def release(self, fh):
393        self.inode_open_count[fh] -= 1
394
395        if self.inode_open_count[fh] == 0:
396            del self.inode_open_count[fh]
397            if (await self.getattr(fh)).st_nlink == 0:
398                self.cursor.execute("DELETE FROM inodes WHERE id=?", (fh,))
399
400class NoUniqueValueError(Exception):
401    def __str__(self):
402        return 'Query generated more than 1 result row'
403
404
405class NoSuchRowError(Exception):
406    def __str__(self):
407        return 'Query produced 0 result rows'
408
409def init_logging(debug=False):
410    formatter = logging.Formatter('%(asctime)s.%(msecs)03d %(threadName)s: '
411                                  '[%(name)s] %(message)s', datefmt="%Y-%m-%d %H:%M:%S")
412    handler = logging.StreamHandler()
413    handler.setFormatter(formatter)
414    root_logger = logging.getLogger()
415    if debug:
416        handler.setLevel(logging.DEBUG)
417        root_logger.setLevel(logging.DEBUG)
418    else:
419        handler.setLevel(logging.INFO)
420        root_logger.setLevel(logging.INFO)
421    root_logger.addHandler(handler)
422
423def parse_args():
424    '''Parse command line'''
425
426    parser = ArgumentParser()
427
428    parser.add_argument('mountpoint', type=str,
429                        help='Where to mount the file system')
430    parser.add_argument('--debug', action='store_true', default=False,
431                        help='Enable debugging output')
432    parser.add_argument('--debug-fuse', action='store_true', default=False,
433                        help='Enable FUSE debugging output')
434
435    return parser.parse_args()
436
437if __name__ == '__main__':
438
439    options = parse_args()
440    init_logging(options.debug)
441    operations = Operations()
442
443    fuse_options = set(pyfuse3.default_options)
444    fuse_options.add('fsname=tmpfs')
445    fuse_options.discard('default_permissions')
446    if options.debug_fuse:
447        fuse_options.add('debug')
448    pyfuse3.init(operations, options.mountpoint, fuse_options)
449
450    try:
451        trio.run(pyfuse3.main)
452    except:
453        pyfuse3.close(unmount=False)
454        raise
455
456    pyfuse3.close()

Passthrough / Overlay File System

(shipped as examples/passthroughfs.py)

  1#!/usr/bin/env python3
  2'''
  3passthroughfs.py - Example file system for pyfuse3
  4
  5This file system mirrors the contents of a specified directory tree.
  6
  7Caveats:
  8
  9 * Inode generation numbers are not passed through but set to zero.
 10
 11 * Block size (st_blksize) and number of allocated blocks (st_blocks) are not
 12   passed through.
 13
 14 * Performance for large directories is not good, because the directory
 15   is always read completely.
 16
 17 * There may be a way to break-out of the directory tree.
 18
 19 * The readdir implementation is not fully POSIX compliant. If a directory
 20   contains hardlinks and is modified during a readdir call, readdir()
 21   may return some of the hardlinked files twice or omit them completely.
 22
 23 * If you delete or rename files in the underlying file system, the
 24   passthrough file system will get confused.
 25
 26Copyright ©  Nikolaus Rath <Nikolaus.org>
 27
 28Permission is hereby granted, free of charge, to any person obtaining a copy of
 29this software and associated documentation files (the "Software"), to deal in
 30the Software without restriction, including without limitation the rights to
 31use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
 32the Software, and to permit persons to whom the Software is furnished to do so.
 33
 34THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 35IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 36FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 37COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 38IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 39CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 40'''
 41
 42import os
 43import sys
 44
 45# If we are running from the pyfuse3 source directory, try
 46# to load the module from there first.
 47basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..'))
 48if (os.path.exists(os.path.join(basedir, 'setup.py')) and
 49    os.path.exists(os.path.join(basedir, 'src', 'pyfuse3.pyx'))):
 50    sys.path.insert(0, os.path.join(basedir, 'src'))
 51
 52import pyfuse3
 53from argparse import ArgumentParser
 54import errno
 55import logging
 56import stat as stat_m
 57from pyfuse3 import FUSEError
 58from os import fsencode, fsdecode
 59from collections import defaultdict
 60import trio
 61
 62import faulthandler
 63faulthandler.enable()
 64
 65log = logging.getLogger(__name__)
 66
 67class Operations(pyfuse3.Operations):
 68
 69    enable_writeback_cache = True
 70
 71    def __init__(self, source):
 72        super().__init__()
 73        self._inode_path_map = { pyfuse3.ROOT_INODE: source }
 74        self._lookup_cnt = defaultdict(lambda : 0)
 75        self._fd_inode_map = dict()
 76        self._inode_fd_map = dict()
 77        self._fd_open_count = dict()
 78
 79    def _inode_to_path(self, inode):
 80        try:
 81            val = self._inode_path_map[inode]
 82        except KeyError:
 83            raise FUSEError(errno.ENOENT)
 84
 85        if isinstance(val, set):
 86            # In case of hardlinks, pick any path
 87            val = next(iter(val))
 88        return val
 89
 90    def _add_path(self, inode, path):
 91        log.debug('_add_path for %d, %s', inode, path)
 92        self._lookup_cnt[inode] += 1
 93
 94        # With hardlinks, one inode may map to multiple paths.
 95        if inode not in self._inode_path_map:
 96            self._inode_path_map[inode] = path
 97            return
 98
 99        val = self._inode_path_map[inode]
100        if isinstance(val, set):
101            val.add(path)
102        elif val != path:
103            self._inode_path_map[inode] = { path, val }
104
105    async def forget(self, inode_list):
106        for (inode, nlookup) in inode_list:
107            if self._lookup_cnt[inode] > nlookup:
108                self._lookup_cnt[inode] -= nlookup
109                continue
110            log.debug('forgetting about inode %d', inode)
111            assert inode not in self._inode_fd_map
112            del self._lookup_cnt[inode]
113            try:
114                del self._inode_path_map[inode]
115            except KeyError: # may have been deleted
116                pass
117
118    async def lookup(self, inode_p, name, ctx=None):
119        name = fsdecode(name)
120        log.debug('lookup for %s in %d', name, inode_p)
121        path = os.path.join(self._inode_to_path(inode_p), name)
122        attr = self._getattr(path=path)
123        if name != '.' and name != '..':
124            self._add_path(attr.st_ino, path)
125        return attr
126
127    async def getattr(self, inode, ctx=None):
128        if inode in self._inode_fd_map:
129            return self._getattr(fd=self._inode_fd_map[inode])
130        else:
131            return self._getattr(path=self._inode_to_path(inode))
132
133    def _getattr(self, path=None, fd=None):
134        assert fd is None or path is None
135        assert not(fd is None and path is None)
136        try:
137            if fd is None:
138                stat = os.lstat(path)
139            else:
140                stat = os.fstat(fd)
141        except OSError as exc:
142            raise FUSEError(exc.errno)
143
144        entry = pyfuse3.EntryAttributes()
145        for attr in ('st_ino', 'st_mode', 'st_nlink', 'st_uid', 'st_gid',
146                     'st_rdev', 'st_size', 'st_atime_ns', 'st_mtime_ns',
147                     'st_ctime_ns'):
148            setattr(entry, attr, getattr(stat, attr))
149        entry.generation = 0
150        entry.entry_timeout = 0
151        entry.attr_timeout = 0
152        entry.st_blksize = 512
153        entry.st_blocks = ((entry.st_size+entry.st_blksize-1) // entry.st_blksize)
154
155        return entry
156
157    async def readlink(self, inode, ctx):
158        path = self._inode_to_path(inode)
159        try:
160            target = os.readlink(path)
161        except OSError as exc:
162            raise FUSEError(exc.errno)
163        return fsencode(target)
164
165    async def opendir(self, inode, ctx):
166        return inode
167
168    async def readdir(self, inode, off, token):
169        path = self._inode_to_path(inode)
170        log.debug('reading %s', path)
171        entries = []
172        for name in os.listdir(path):
173            if name == '.' or name == '..':
174                continue
175            attr = self._getattr(path=os.path.join(path, name))
176            entries.append((attr.st_ino, name, attr))
177
178        log.debug('read %d entries, starting at %d', len(entries), off)
179
180        # This is not fully posix compatible. If there are hardlinks
181        # (two names with the same inode), we don't have a unique
182        # offset to start in between them. Note that we cannot simply
183        # count entries, because then we would skip over entries
184        # (or return them more than once) if the number of directory
185        # entries changes between two calls to readdir().
186        for (ino, name, attr) in sorted(entries):
187            if ino <= off:
188                continue
189            if not pyfuse3.readdir_reply(
190                token, fsencode(name), attr, ino):
191                break
192            self._add_path(attr.st_ino, os.path.join(path, name))
193
194    async def unlink(self, inode_p, name, ctx):
195        name = fsdecode(name)
196        parent = self._inode_to_path(inode_p)
197        path = os.path.join(parent, name)
198        try:
199            inode = os.lstat(path).st_ino
200            os.unlink(path)
201        except OSError as exc:
202            raise FUSEError(exc.errno)
203        if inode in self._lookup_cnt:
204            self._forget_path(inode, path)
205
206    async def rmdir(self, inode_p, name, ctx):
207        name = fsdecode(name)
208        parent = self._inode_to_path(inode_p)
209        path = os.path.join(parent, name)
210        try:
211            inode = os.lstat(path).st_ino
212            os.rmdir(path)
213        except OSError as exc:
214            raise FUSEError(exc.errno)
215        if inode in self._lookup_cnt:
216            self._forget_path(inode, path)
217
218    def _forget_path(self, inode, path):
219        log.debug('forget %s for %d', path, inode)
220        val = self._inode_path_map[inode]
221        if isinstance(val, set):
222            val.remove(path)
223            if len(val) == 1:
224                self._inode_path_map[inode] = next(iter(val))
225        else:
226            del self._inode_path_map[inode]
227
228    async def symlink(self, inode_p, name, target, ctx):
229        name = fsdecode(name)
230        target = fsdecode(target)
231        parent = self._inode_to_path(inode_p)
232        path = os.path.join(parent, name)
233        try:
234            os.symlink(target, path)
235            os.chown(path, ctx.uid, ctx.gid, follow_symlinks=False)
236        except OSError as exc:
237            raise FUSEError(exc.errno)
238        stat = os.lstat(path)
239        self._add_path(stat.st_ino, path)
240        return await self.getattr(stat.st_ino)
241
242    async def rename(self, inode_p_old, name_old, inode_p_new, name_new,
243                     flags, ctx):
244        if flags != 0:
245            raise FUSEError(errno.EINVAL)
246
247        name_old = fsdecode(name_old)
248        name_new = fsdecode(name_new)
249        parent_old = self._inode_to_path(inode_p_old)
250        parent_new = self._inode_to_path(inode_p_new)
251        path_old = os.path.join(parent_old, name_old)
252        path_new = os.path.join(parent_new, name_new)
253        try:
254            os.rename(path_old, path_new)
255            inode = os.lstat(path_new).st_ino
256        except OSError as exc:
257            raise FUSEError(exc.errno)
258        if inode not in self._lookup_cnt:
259            return
260
261        val = self._inode_path_map[inode]
262        if isinstance(val, set):
263            assert len(val) > 1
264            val.add(path_new)
265            val.remove(path_old)
266        else:
267            assert val == path_old
268            self._inode_path_map[inode] = path_new
269
270    async def link(self, inode, new_inode_p, new_name, ctx):
271        new_name = fsdecode(new_name)
272        parent = self._inode_to_path(new_inode_p)
273        path = os.path.join(parent, new_name)
274        try:
275            os.link(self._inode_to_path(inode), path, follow_symlinks=False)
276        except OSError as exc:
277            raise FUSEError(exc.errno)
278        self._add_path(inode, path)
279        return await self.getattr(inode)
280
281    async def setattr(self, inode, attr, fields, fh, ctx):
282        # We use the f* functions if possible so that we can handle
283        # a setattr() call for an inode without associated directory
284        # handle.
285        if fh is None:
286            path_or_fh = self._inode_to_path(inode)
287            truncate = os.truncate
288            chmod = os.chmod
289            chown = os.chown
290            stat = os.lstat
291        else:
292            path_or_fh = fh
293            truncate = os.ftruncate
294            chmod = os.fchmod
295            chown = os.fchown
296            stat = os.fstat
297
298        try:
299            if fields.update_size:
300                truncate(path_or_fh, attr.st_size)
301
302            if fields.update_mode:
303                # Under Linux, chmod always resolves symlinks so we should
304                # actually never get a setattr() request for a symbolic
305                # link.
306                assert not stat_m.S_ISLNK(attr.st_mode)
307                chmod(path_or_fh, stat_m.S_IMODE(attr.st_mode))
308
309            if fields.update_uid:
310                chown(path_or_fh, attr.st_uid, -1, follow_symlinks=False)
311
312            if fields.update_gid:
313                chown(path_or_fh, -1, attr.st_gid, follow_symlinks=False)
314
315            if fields.update_atime and fields.update_mtime:
316                if fh is None:
317                    os.utime(path_or_fh, None, follow_symlinks=False,
318                             ns=(attr.st_atime_ns, attr.st_mtime_ns))
319                else:
320                    os.utime(path_or_fh, None,
321                             ns=(attr.st_atime_ns, attr.st_mtime_ns))
322            elif fields.update_atime or fields.update_mtime:
323                # We can only set both values, so we first need to retrieve the
324                # one that we shouldn't be changing.
325                oldstat = stat(path_or_fh)
326                if not fields.update_atime:
327                    attr.st_atime_ns = oldstat.st_atime_ns
328                else:
329                    attr.st_mtime_ns = oldstat.st_mtime_ns
330                if fh is None:
331                    os.utime(path_or_fh, None, follow_symlinks=False,
332                             ns=(attr.st_atime_ns, attr.st_mtime_ns))
333                else:
334                    os.utime(path_or_fh, None,
335                             ns=(attr.st_atime_ns, attr.st_mtime_ns))
336
337        except OSError as exc:
338            raise FUSEError(exc.errno)
339
340        return await self.getattr(inode)
341
342    async def mknod(self, inode_p, name, mode, rdev, ctx):
343        path = os.path.join(self._inode_to_path(inode_p), fsdecode(name))
344        try:
345            os.mknod(path, mode=(mode & ~ctx.umask), device=rdev)
346            os.chown(path, ctx.uid, ctx.gid)
347        except OSError as exc:
348            raise FUSEError(exc.errno)
349        attr = self._getattr(path=path)
350        self._add_path(attr.st_ino, path)
351        return attr
352
353    async def mkdir(self, inode_p, name, mode, ctx):
354        path = os.path.join(self._inode_to_path(inode_p), fsdecode(name))
355        try:
356            os.mkdir(path, mode=(mode & ~ctx.umask))
357            os.chown(path, ctx.uid, ctx.gid)
358        except OSError as exc:
359            raise FUSEError(exc.errno)
360        attr = self._getattr(path=path)
361        self._add_path(attr.st_ino, path)
362        return attr
363
364    async def statfs(self, ctx):
365        root = self._inode_path_map[pyfuse3.ROOT_INODE]
366        stat_ = pyfuse3.StatvfsData()
367        try:
368            statfs = os.statvfs(root)
369        except OSError as exc:
370            raise FUSEError(exc.errno)
371        for attr in ('f_bsize', 'f_frsize', 'f_blocks', 'f_bfree', 'f_bavail',
372                     'f_files', 'f_ffree', 'f_favail'):
373            setattr(stat_, attr, getattr(statfs, attr))
374        stat_.f_namemax = statfs.f_namemax - (len(root)+1)
375        return stat_
376
377    async def open(self, inode, flags, ctx):
378        if inode in self._inode_fd_map:
379            fd = self._inode_fd_map[inode]
380            self._fd_open_count[fd] += 1
381            return pyfuse3.FileInfo(fh=fd)
382        assert flags & os.O_CREAT == 0
383        try:
384            fd = os.open(self._inode_to_path(inode), flags)
385        except OSError as exc:
386            raise FUSEError(exc.errno)
387        self._inode_fd_map[inode] = fd
388        self._fd_inode_map[fd] = inode
389        self._fd_open_count[fd] = 1
390        return pyfuse3.FileInfo(fh=fd)
391
392    async def create(self, inode_p, name, mode, flags, ctx):
393        path = os.path.join(self._inode_to_path(inode_p), fsdecode(name))
394        try:
395            fd = os.open(path, flags | os.O_CREAT | os.O_TRUNC)
396        except OSError as exc:
397            raise FUSEError(exc.errno)
398        attr = self._getattr(fd=fd)
399        self._add_path(attr.st_ino, path)
400        self._inode_fd_map[attr.st_ino] = fd
401        self._fd_inode_map[fd] = attr.st_ino
402        self._fd_open_count[fd] = 1
403        return (pyfuse3.FileInfo(fh=fd), attr)
404
405    async def read(self, fd, offset, length):
406        os.lseek(fd, offset, os.SEEK_SET)
407        return os.read(fd, length)
408
409    async def write(self, fd, offset, buf):
410        os.lseek(fd, offset, os.SEEK_SET)
411        return os.write(fd, buf)
412
413    async def release(self, fd):
414        if self._fd_open_count[fd] > 1:
415            self._fd_open_count[fd] -= 1
416            return
417
418        del self._fd_open_count[fd]
419        inode = self._fd_inode_map[fd]
420        del self._inode_fd_map[inode]
421        del self._fd_inode_map[fd]
422        try:
423            os.close(fd)
424        except OSError as exc:
425            raise FUSEError(exc.errno)
426
427def init_logging(debug=False):
428    formatter = logging.Formatter('%(asctime)s.%(msecs)03d %(threadName)s: '
429                                  '[%(name)s] %(message)s', datefmt="%Y-%m-%d %H:%M:%S")
430    handler = logging.StreamHandler()
431    handler.setFormatter(formatter)
432    root_logger = logging.getLogger()
433    if debug:
434        handler.setLevel(logging.DEBUG)
435        root_logger.setLevel(logging.DEBUG)
436    else:
437        handler.setLevel(logging.INFO)
438        root_logger.setLevel(logging.INFO)
439    root_logger.addHandler(handler)
440
441
442def parse_args(args):
443    '''Parse command line'''
444
445    parser = ArgumentParser()
446
447    parser.add_argument('source', type=str,
448                        help='Directory tree to mirror')
449    parser.add_argument('mountpoint', type=str,
450                        help='Where to mount the file system')
451    parser.add_argument('--debug', action='store_true', default=False,
452                        help='Enable debugging output')
453    parser.add_argument('--debug-fuse', action='store_true', default=False,
454                        help='Enable FUSE debugging output')
455
456    return parser.parse_args(args)
457
458def main():
459    options = parse_args(sys.argv[1:])
460    init_logging(options.debug)
461    operations = Operations(options.source)
462
463    log.debug('Mounting...')
464    fuse_options = set(pyfuse3.default_options)
465    fuse_options.add('fsname=passthroughfs')
466    if options.debug_fuse:
467        fuse_options.add('debug')
468    pyfuse3.init(operations, options.mountpoint, fuse_options)
469
470    try:
471        log.debug('Entering main loop..')
472        trio.run(pyfuse3.main)
473    except:
474        pyfuse3.close(unmount=False)
475        raise
476
477    log.debug('Unmounting..')
478    pyfuse3.close()
479
480if __name__ == '__main__':
481    main()