Pset 4 for 2019.

GNUmakefile

@@ -44,6 +44,7 @@ KERNEL_OBJS = $(OBJDIR)/k-exception.ko \
 	$(OBJDIR)/k-alloc.ko $(OBJDIR)/k-vmiter.ko $(OBJDIR)/k-devices.ko \
 	$(OBJDIR)/k-init.ko $(OBJDIR)/k-hardware.ko $(OBJDIR)/k-mpspec.ko \
 	$(OBJDIR)/crc32c.ko \
+	$(OBJDIR)/k-ahci.ko $(OBJDIR)/k-chkfs.ko $(OBJDIR)/k-chkfsiter.ko \
 	$(OBJDIR)/k-memviewer.ko $(OBJDIR)/lib.ko $(OBJDIR)/k-initfs.ko
 
 PROCESSES = $(patsubst %.cc,%,$(wildcard p-*.cc))

build/chickadeefsck.cc

@@ -8,6 +8,7 @@
 #include <inttypes.h>
 #include <vector>
 #include <deque>
+#include <string>
 #include <unordered_set>
 #include <algorithm>
 #include <stdarg.h>
@@ -22,11 +23,12 @@
 static bool verbose = false;
 static size_t nerrors = 0;
 static size_t nwarnings = 0;
+static const char* extract = nullptr;
 
 static void eprintf(const char* format, ...) {
     va_list val;
     va_start(val, format);
-    vfprintf(stdout, format, val);
+    vfprintf(extract ? stderr : stdout, format, val);
     va_end(val);
     ++nerrors;
 }
@@ -34,7 +36,7 @@ static void eprintf(const char* format, ...) {
 static void ewprintf(const char* format, ...) {
     va_list val;
     va_start(val, format);
-    vfprintf(stdout, format, val);
+    vfprintf(extract ? stderr : stdout, format, val);
     va_end(val);
     ++nwarnings;
 }
@@ -42,13 +44,12 @@ static void ewprintf(const char* format, ...) {
 static void exprintf(const char* format, ...) {
     va_list val;
     va_start(val, format);
-    vfprintf(stdout, format, val);
+    vfprintf(extract ? stderr : stdout, format, val);
     va_end(val);
 }
 
 
 static constexpr size_t blocksize = chkfs::blocksize;
-static constexpr size_t inodesize = sizeof(chkfs::inode);
 using blocknum_t = chkfs::blocknum_t;
 using inum_t = chkfs::inum_t;
 
@@ -58,12 +59,12 @@ static chkfs::superblock sb;
 
 enum blocktype {
     bunused = 0, bsuperblock, bswap, bfbb, binode,
-    bjournal, bfree, bdirectory, bdata, bindirect, bindirect2
+    bjournal, bfree, bdirectory, bdata, bindirect
 };
 
 static const char* const typenames[] = {
     "unused", "superblock", "swap", "fbb", "inode",
-    "journal", "free", "directory", "data", "indirect", "indirect2"
+    "journal", "free", "directory", "data", "indirect"
 };
 
 struct blockinfo {
@@ -130,9 +131,7 @@ struct inodeinfo {
     void visit(const char* ref);
     void scan();
     void finish_visit();
-    void visit_data(blocknum_t b, size_t idx, size_t sz);
-    void visit_indirect(blocknum_t b, size_t idx, size_t sz);
-    void visit_indirect2(blocknum_t b, size_t idx, size_t sz);
+    unsigned visit_data(blocknum_t b, blocknum_t count, unsigned bi, size_t sz);
     void visit_directory_data(blocknum_t b, size_t pos, size_t sz);
     unsigned char* get_data_block(unsigned bi);
     inum_t lookup(const char* name);
@@ -210,14 +209,15 @@ void inodeinfo::finish_visit() {
             sprintf(typebuf, "<type %d>", from_le(in->type));
             type = typebuf;
         }
-        printf("inode %u @%s: size %zu, type %s, nlink %u\n",
-               inum, ref_, sz, type, from_le(in->nlink));
+        char indirbuf[100] = "";
+        if (in->indirect.first || in->indirect.count) {
+            sprintf(indirbuf, ", indirect extent %u+%u",
+                    from_le(in->indirect.first), from_le(in->indirect.count));
+        }
+        printf("inode %u @%s: size %zu, type %s, nlink %u%s\n",
+               inum, ref_, sz, type, from_le(in->nlink), indirbuf);
     }
 
-    if (sz > chkfs::maxindirect2size) {
-        eprintf("inode %u @%s: size %zu too big (max %zu)\n",
-                inum, ref_, sz, chkfs::maxindirect2size);
-    }
     if (type_ == bdirectory) {
         contents_ = new std::unordered_set<std::string>();
         if (sz % sizeof(chkfs::dirent) != 0) {
@@ -226,40 +226,84 @@ void inodeinfo::finish_visit() {
         }
     }
 
-    for (size_t i = 0; i != chkfs::ndirect; ++i) {
-        visit_data(from_le(in->direct[i]), i, sz);
-    }
-    visit_indirect(from_le(in->indirect), chkfs::ndirect, sz);
-    visit_indirect2(from_le(in->indirect2),
-                    chkfs::ndirect + chkfs::nindirect, sz);
-
-    delete contents_;
-}
-
-void inodeinfo::visit_data(blocknum_t b, size_t idx, size_t sz) {
-    if (b != 0) {
-        if (verbose) {
-            printf("  [%zu]: data block %u\n", idx, b);
+    chkfs::extent* end_indir = nullptr;
+    size_t pos = 0;
+    bool saw_empty = false;
+    for (chkfs::extent* e = &in->direct[0]; true; ++e) {
+        if (e == &in->indirect) {
+            e = nullptr;
+            blocknum_t first = from_le(in->indirect.first);
+            blocknum_t count = from_le(in->indirect.count);
+            if (first && count) {
+                if (first >= sb.data_bn
+                    && first < sb.journal_bn
+                    && first + count <= sb.journal_bn) {
+                    e = reinterpret_cast<chkfs::extent*>(data + first * blocksize);
+                    end_indir = reinterpret_cast<chkfs::extent*>(data + (first + count) * blocksize);
+                    for (blocknum_t bb = 0; bb != count; ++bb) {
+                        blockinfo::blocks[first + bb].visit(bindirect, ref_, bb);
+                    }
+                } else {
+                    eprintf("inode %u @%s: indirect extent %u+%u out of range\n",
+                            inum, ref_, first, count);
+                }
+            } else {
+                if (!first && count) {
+                    eprintf("inode %u @%s: nonempty indirect extent starts at zero block\n",
+                            inum, ref_);
+                }
+                if (pos < sz) {
+                    eprintf("inode %u @%s: missing indirect block\n",
+                            inum, ref_);
+                }
+            }
+        }
+        if (e == end_indir) {
+            break;
         }
-        if (idx * blocksize >= sz) {
-            ewprintf("inode %u @%s [%zu]: warning: dangling block reference\n",
-                     get_inum(), ref_, idx);
+
+        blocknum_t first = from_le(e->first);
+        blocknum_t count = from_le(e->count);
+        if (count && saw_empty) {
+            eprintf("inode %u @%s [%zu]: nonempty extent follows empty extent\n",
+                    inum, ref_, pos / blocksize);
         }
-        if (b < blockinfo::blocks.size()) {
-            blockinfo::blocks[b].visit(type_, ref_, idx);
-            if (type_ == bdirectory) {
-                visit_directory_data(b, idx * blocksize, sz);
+        if (first && count) {
+            if (verbose) {
+                printf("  [%zu]: extent %u+%u\n", pos / blocksize, first, count);
+            }
+            for (blocknum_t bb = 0; bb != count; ++bb, pos += blocksize) {
+                if (pos > sz) {
+                    ewprintf("inode %u @%s [%zu]: warning: dangling block reference\n",
+                             inum, ref_, pos / blocksize);
+                }
+                if (first + bb < blockinfo::blocks.size()) {
+                    blockinfo::blocks[first + bb].visit(type_, ref_, pos / blocksize);
+                    if (type_ == bdirectory) {
+                        visit_directory_data(first + bb, pos, sz);
+                    }
+                } else {
+                    eprintf("inode %u @%s [%zu]: block number %u out of range\n",
+                            inum, ref_, pos / blocksize, first + bb);
+                }
             }
         } else {
-            eprintf("inode %u @%s [%zu]: block number %u out of range\n",
-                    get_inum(), ref_, idx, b);
-        }
-    } else {
-        if (idx * blocksize < sz) {
-            ewprintf("inode %u @%s [%zu]: warning: hole in file\n",
-                     get_inum(), ref_, idx);
+            if (!first && count) {
+                eprintf("inode %u @%s [%zu]: nonempty extent starts at zero block\n",
+                        inum, ref_, pos / blocksize);
+            }
+            if (count && pos < sz) {
+                eprintf("inode %u @%s [%zu]: warning: hole in file\n",
+                        inum, ref_, pos / blocksize);
+            }
+            if (!count) {
+                saw_empty = true;
+            }
+            pos += count * blocksize;
         }
     }
+
+    delete contents_;
 }
 
 void inodeinfo::visit_directory_data(blocknum_t b, size_t pos, size_t sz) {
@@ -312,104 +356,39 @@ void inodeinfo::visit_directory_data(blocknum_t b, size_t pos, size_t sz) {
     }
 }
 
-void inodeinfo::visit_indirect(blocknum_t b, size_t idx, size_t sz) {
-    if (b != 0) {
-        if (verbose) {
-            printf("  [%zu]: indirect block %u\n", idx, b);
-        }
-        if (idx * blocksize >= sz) {
-            ewprintf("inode %u @%s [%zu]: warning: dangling indirect block reference\n",
-                     get_inum(), ref_, idx);
-        }
-        if (b < blockinfo::blocks.size()) {
-            blockinfo::blocks[b].visit(bindirect, ref_, idx);
-            blocknum_t* bdata = reinterpret_cast<blocknum_t*>
-                (data + b * blocksize);
-            for (size_t i = 0; i != chkfs::nindirect; ++i) {
-                visit_data(from_le(bdata[i]), idx + i, sz);
-            }
-        } else {
-            eprintf("inode %u @%s [%zu]: block number %u out of range\n",
-                    get_inum(), ref_, idx, b);
-        }
-    } else {
-        if (idx * blocksize < sz) {
-            ewprintf("inode %u @%s [%zu]: warning: %s\n",
-                     get_inum(), ref_, idx,
-                     idx == chkfs::ndirect ? "missing indirect block"
-                     : "hole in file");
-        }
-    }
-}
-
-void inodeinfo::visit_indirect2(blocknum_t b, size_t idx, size_t sz) {
-    if (b != 0) {
-        if (verbose) {
-            printf("  [%zu]: indirect2 block %u\n", idx, b);
-        }
-        if (idx * blocksize >= sz) {
-            ewprintf("inode %u @%s [%zu]: warning: dangling indirect2 block reference\n",
-                     get_inum(), ref_, idx);
-        }
-        if (b < blockinfo::blocks.size()) {
-            blockinfo::blocks[b].visit(bindirect2, ref_, idx);
-            blocknum_t* bdata = reinterpret_cast<blocknum_t*>
-                (data + b * blocksize);
-            for (size_t i = 0; i != chkfs::nindirect; ++i) {
-                visit_indirect(from_le(bdata[i]), idx + i * chkfs::nindirect, sz);
-            }
-        } else {
-            eprintf("inode %u @%s [%zu]: block number %u out of range\n",
-                    get_inum(), ref_, idx, b);
-        }
-    } else {
-        if (idx * blocksize < sz) {
-            ewprintf("inode %u @%s [%zu]: warning: %s\n",
-                     get_inum(), ref_, idx,
-                     idx == chkfs::ndirect + chkfs::nindirect
-                     ? "missing indirect2 block"
-                     : "hole in file");
-        }
-    }
-}
-
 
 unsigned char* inodeinfo::get_data_block(unsigned bi) {
     auto in = get_inode();
 
-    blocknum_t indir;
-    if (bi >= chkfs::ndirect + chkfs::nindirect) {
-        if (in->indirect2 < sb.data_bn
-            || in->indirect2 >= sb.journal_bn) {
+    chkfs::extent* end_indir = nullptr;
+    for (chkfs::extent* e = &in->direct[0]; true; ++e) {
+        if (e == &in->indirect) {
+            blocknum_t first = from_le(in->indirect.first);
+            blocknum_t count = from_le(in->indirect.count);
+            if (first < sb.data_bn
+                || !count
+                || first + count > sb.journal_bn) {
+                return nullptr;
+            }
+            e = reinterpret_cast<chkfs::extent*>(data + first * blocksize);
+            end_indir = reinterpret_cast<chkfs::extent*>(data + (first + count) * blocksize);
+        }
+        if (e == end_indir) {
             return nullptr;
         }
-        blocknum_t* indir2d = reinterpret_cast<blocknum_t*>
-            (data + in->indirect2 * blocksize);
-        indir = indir2d[chkfs::bi_indirect_index(bi)];
-    } else if (bi >= chkfs::ndirect) {
-        indir = in->indirect;
-    } else {
-        indir = 0;
-    }
-
-    blocknum_t dir;
-    if (bi >= chkfs::ndirect) {
-        if (indir < sb.data_bn
-            || indir >= sb.journal_bn) {
+        blocknum_t first = from_le(e->first);
+        blocknum_t count = from_le(e->count);
+        if (bi < count) {
+            if (first < sb.data_bn
+                || first + count > sb.journal_bn) {
+                return nullptr;
+            } else {
+                return data + (first + bi) * blocksize;
+            }
+        } else if (count == 0) {
             return nullptr;
         }
-        blocknum_t* indird = reinterpret_cast<blocknum_t*>
-            (data + indir * blocksize);
-        dir = indird[chkfs::bi_direct_index(bi)];
-    } else {
-        dir = in->direct[bi];
-    }
-
-    if (dir < sb.data_bn
-        || dir >= sb.journal_bn) {
-        return nullptr;
-    } else {
-        return data + dir * blocksize;
+        bi -= count;
     }
 }
 
@@ -534,7 +513,6 @@ static struct option options[] = {
 int main(int argc, char** argv) {
     bool replay = false;
     bool no_journal = false;
-    const char* extract = nullptr;
 
     int opt;
     while ((opt = getopt_long(argc, argv, "Vse:", options, nullptr)) != -1) {

build/mkchickadeefs.cc

@@ -31,6 +31,11 @@ static unsigned char** blocks;
 static unsigned freeb;
 static chkfs::inum_t freeinode;
 static std::vector<chkfs::dirent> root;
+static bool randomize;
+static uint32_t first_datab = 0;
+static std::vector<chkfs::extent> extents;
+static std::default_random_engine engine;
+static std::discrete_distribution<uint32_t> extentsize_distribution {0, 3, 2, 1, 1, 1, 1};
 
 
 inline void mark_free(blocknum_t bnum) {
@@ -90,9 +95,9 @@ static chkfs::inode* get_inode(chkfs::inum_t inum) {
 static chkfs::inum_t
 add_inode(chkfs::inum_t inum,
           unsigned type, size_t sz, unsigned nlink,
-          blocknum_t first_block, const char* path,
-          bool indirect_at_end = false) {
-    assert(freeb >= first_block + (sz + blocksize - 1) / blocksize);
+          blocknum_t first_block, const char* path) {
+    uint32_t nblocks = (sz + blocksize - 1) / blocksize;
+    assert(freeb >= first_block + nblocks);
 
     if (inum == 0) {
         if (freeinode == sb.ninodes) {
@@ -108,42 +113,45 @@ add_inode(chkfs::inum_t inum,
     ino->size = to_le(sz);
     ino->nlink = to_le(nlink);
 
-    // allocate indirect block
-    if (sz > chkfs::maxindirectsize) {
-        fprintf(stderr, "%s: file too big for indirect block\n", path);
-        exit(1);
-    }
-    blocknum_t* indir = nullptr;
-    if (sz > chkfs::maxdirectsize) {
-        advance_blockno(path);
-        indir = new blocknum_t[chkfs::nindirect];
-        memset(indir, 0, blocksize);
-        if (indirect_at_end) {
-            ino->indirect = to_le(freeb - 1);
-            blocks[freeb - 1] = reinterpret_cast<unsigned char*>(indir);
+    chkfs::extent* indir = nullptr;
+    size_t extenti = 0;
+    for (uint32_t eb = 0; eb < nblocks; ) {
+        uint32_t nb;
+        if (randomize
+            && (!first_datab || first_datab != first_block)) {
+            nb = extentsize_distribution(engine);
         } else {
-            ++first_block;
-            memmove(&blocks[first_block], &blocks[first_block - 1],
-                    sizeof(unsigned char*) * (freeb - first_block));
-            ino->indirect = to_le(first_block - 1);
-            blocks[first_block - 1] = reinterpret_cast<unsigned char*>(indir);
+            nb = nblocks;
         }
-    }
+        nb = std::min(nb, nblocks - eb);
 
-    // assign block pointers
-    for (size_t bidx = 0; bidx * blocksize < sz; ++bidx) {
-        if (bidx < chkfs::ndirect) {
-            ino->direct[bidx] = to_le(first_block + bidx);
+        chkfs::extent* ex;
+        if (extenti < chkfs::ndirect) {
+            ex = &ino->direct[extenti];
         } else {
-            indir[bidx - chkfs::ndirect] = to_le(first_block + bidx);
+            if (!indir) {
+                advance_blockno(path);
+                indir = new chkfs::extent[chkfs::extentsperblock];
+                memset(indir, 0, blocksize);
+                blocks[freeb - 1] = reinterpret_cast<unsigned char*>(indir);
+                ino->indirect.first = to_le(freeb - 1);
+                ino->indirect.count = to_le(uint32_t(1));
+                extents.push_back(chkfs::extent{freeb - 1, 1});
+            }
+            ex = &indir[extenti - chkfs::ndirect];
         }
+        ex->first = to_le(first_block + eb);
+        ex->count = to_le(nb);
+
+        extents.push_back(chkfs::extent{first_block + eb, nb});
+        ++extenti;
+        eb += nb;
     }
 
     return inum;
 }
 
-static void add_file(const char* path, const char* name,
-                     bool indirect_at_end) {
+static void add_file(const char* path, const char* name) {
     FILE* f = fopencheck(path);
     size_t sz = 0;
     uint32_t first_block = freeb;
@@ -173,8 +181,7 @@ static void add_file(const char* path, const char* name,
 
     // allocate inode
     uint32_t ino = add_inode
-        (0, chkfs::type_regular, sz, 1, first_block, path,
-         indirect_at_end);
+        (0, chkfs::type_regular, sz, 1, first_block, path);
 
     // add to directory
     if (strcmp(name, ".") == 0
@@ -192,7 +199,7 @@ static void add_file(const char* path, const char* name,
 }
 
 
-static void shuffle_blocks(bool preserve_inode2);
+static void shuffle_blocks();
 
 static void parse_uint32(const char* arg, uint32_t* val, int opt) {
     unsigned long n;
@@ -238,10 +245,8 @@ Create a ChickadeeFS image from the arguments.\n\
 }
 
 int main(int argc, char** argv) {
-    uint32_t first_datab = 0;
     const char* bootsector = nullptr;
     const char* outfile = nullptr;
-    bool randomize = false;
 
     int opt;
     while ((opt = getopt_long(argc, argv, "b:i:w:j:f:rs:o:",
@@ -362,7 +367,6 @@ int main(int argc, char** argv) {
     freeinode = 2;
 
     // read files
-    bool is_first = !!first_datab;
     for (; optind < argc; ++optind) {
         char* colon = strchr(argv[optind], ':');
         const char* name = argv[optind];
@@ -380,8 +384,7 @@ int main(int argc, char** argv) {
                 name += 7;
             }
         }
-        add_file(argv[optind], name, is_first);
-        is_first = false;
+        add_file(argv[optind], name);
     }
 
     // add root directory
@@ -415,7 +418,7 @@ int main(int argc, char** argv) {
 
     // randomize if requested
     if (randomize) {
-        shuffle_blocks(first_datab != 0);
+        shuffle_blocks();
     }
 
     // write output
@@ -467,63 +470,43 @@ int main(int argc, char** argv) {
 
 
 static void shuffle_indirect(unsigned char* data, blocknum_t* perm) {
-    blocknum_t* indir = reinterpret_cast<blocknum_t*>(data);
-    for (int i = 0; i != chkfs::nindirect; ++i) {
-        indir[i] = to_le(perm[from_le(indir[i])]);
-    }
-}
-
-static void shuffle_indirect2(unsigned char* data, blocknum_t* perm) {
-    blocknum_t* indir2 = reinterpret_cast<blocknum_t*>(data);
-    for (int i = 0; i != chkfs::nindirect; ++i) {
-        shuffle_indirect(blocks[from_le(indir2[i])], perm);
-        indir2[i] = to_le(perm[from_le(indir2[i])]);
+    chkfs::extent* indir = reinterpret_cast<chkfs::extent*>(data);
+    for (int i = 0; i != chkfs::extentsperblock; ++i) {
+        indir[i].first = to_le(perm[from_le(indir[i].first)]);
     }
 }
 
 static void shuffle_inode(chkfs::inode* in, blocknum_t* perm) {
     for (size_t i = 0; i != chkfs::ndirect; ++i) {
-        in->direct[i] = to_le(perm[from_le(in->direct[i])]);
+        in->direct[i].first = to_le(perm[from_le(in->direct[i].first)]);
     }
-    if (from_le(in->indirect)) {
-        shuffle_indirect(blocks[from_le(in->indirect)], perm);
-        in->indirect = to_le(perm[from_le(in->indirect)]);
-    }
-    if (from_le(in->indirect2)) {
-        shuffle_indirect2(blocks[from_le(in->indirect2)], perm);
-        in->indirect2 = to_le(perm[from_le(in->indirect2)]);
+    if (from_le(in->indirect.first)) {
+        shuffle_indirect(blocks[from_le(in->indirect.first)], perm);
+        in->indirect.first = to_le(perm[from_le(in->indirect.first)]);
     }
 }
 
-static void shuffle_blocks(bool preserve_inode2) {
-    std::vector<blocknum_t> perm;
-    std::vector<blocknum_t> shufflein;
-    std::vector<blocknum_t> shuffleout;
-
-    size_t data_bn = sb.data_bn;
-    if (preserve_inode2) {
-        chkfs::inode* in = get_inode(2);
-        if (from_le(in->size) != 0) {
-            assert(from_le(in->direct[0]) == data_bn);
-            data_bn += (from_le(in->size) + blocksize - 1) / blocksize;
+static void shuffle_blocks() {
+    std::vector<blocknum_t> perm(sb.nblocks);
+    std::iota(perm.begin(), perm.end(), blocknum_t(0));
+
+    // shuffle extents
+    std::shuffle(extents.begin() + (first_datab && sb.data_bn == first_datab),
+                 extents.end(), engine);
+    std::discrete_distribution<int> space_distribution {13, 2, 1};
+    size_t bn = sb.data_bn;
+    for (auto it = extents.begin(); it != extents.end(); ++it) {
+        if (it != extents.begin()) {
+            size_t nspace = std::min(sb.journal_bn - freeb, blocknum_t(space_distribution(engine)));
+            for (uint32_t bi = 0; bi != nspace; ++bi, ++bn, ++freeb) {
+                perm[freeb] = bn;
+            }
         }
-    }
-
-    for (blocknum_t b = 0; b != sb.nblocks; ++b) {
-        perm.push_back(b);
-        if (b >= data_bn && b < sb.journal_bn && b < freeb + 128) {
-            shufflein.push_back(b);
-            shuffleout.push_back(b);
+        for (uint32_t bi = 0; bi != it->count; ++bi, ++bn) {
+            perm[it->first + bi] = bn;
         }
     }
 
-    // shuffle it
-    std::shuffle(shuffleout.begin(), shuffleout.end(),
-                 std::default_random_engine());
-    for (size_t i = 0; i != shufflein.size(); ++i) {
-        perm[shufflein[i]] = shuffleout[i];
-    }
-
     // shuffle inodes
     for (chkfs::inum_t i = 1; i != freeinode; ++i) {
         shuffle_inode(get_inode(i), perm.data());
@@ -556,3 +539,5 @@ static_assert(sizeof(chkfs::inode) == chkfs::inodesize,
               "inodesize valid");
 static_assert(sizeof(chkfs::dirent) == chkfs::direntsize,
               "direntsize valid");
+static_assert(sizeof(chkfs::extent) == chkfs::extentsize,
+              "extentsize valid");

chickadeefs.hh

@@ -5,12 +5,13 @@
 # include "lib.hh"