feat: s3-compatible vault support

.gitignore

@@ -10,6 +10,7 @@ lerna-debug.log*
 node_modules
 dist
 dist-ssr
+dist-vault
 *.local
 
 # Editor directories and files

config.yaml

@@ -3,8 +3,8 @@ description: an example web application
 language: en
 
 vault:
-  root: public
+  type: file
-  cleanup: false
+  path: /path/to/vault
 
 metadata:
   hidden:

package.json

@@ -6,7 +6,7 @@
   "scripts": {
     "dev": "vite",
     "build": "vite build",
-    "clean": "rm -rf ./dist",
+    "clean": "rm -rf ./dist*",
     "update": "node ./src/scripts/sprachbund.js",
     "preview": "vite preview"
   },
@@ -14,6 +14,7 @@
     "@tailwindcss/typography": "^0.5.15",
     "autoprefixer": "^10.4.20",
     "front-matter": "^4.0.2",
+    "minio": "^8.0.2",
     "mustache": "^4.2.0",
     "postcss": "^8.4.45",
     "tailwindcss": "^3.4.10",

public/.gitignore

@@ -1,2 +0,0 @@
-*
-!.gitignore

src/paths.js

@@ -6,6 +6,7 @@ export const PATH = {};
 
 PATH.CONFIG   = 'config.yaml';
 PATH.OUTPUT   = 'dist';
+PATH.S3TEMP   = 'dist-vault';
 PATH.OBJECT   = `${NATIVE ? PATH.OUTPUT : ''}/_site`;
 PATH.INDEX    = `${PATH.OBJECT}/index`;
 PATH.METADATA = `${PATH.OBJECT}/metadata`;

src/scripts/sprachbund.js

@@ -2,12 +2,13 @@ import { PATH } from '../paths.js'
 import { transform } from './zettelkasten.js'
 import { Index, Attachment, Document } from './object.js'
 
-import { createHash } from 'node:crypto'
+import { Client } from 'minio'
+import { createHash } from 'crypto'
 import { encode, decode } from '@msgpack/msgpack'
 
 import Mustache from 'mustache'
 import YAML from 'yaml'
-import fs from 'node:fs'
+import fs from 'fs'
 import fm from'front-matter'
 
 
@@ -139,8 +140,7 @@ function buildObject(index, ctx) /* always return falsy values */ {
     fs.writeFileSync(path, content);
 }
 
-function buildIndex(config) {
+function buildIndex(metadata, vault) {
-    let metadata = Index.Metadata(config["metadata"]);
     let index = new Index(metadata, {});
 
     if (! fs.existsSync(PATH.INDEX)) {
@@ -148,26 +148,17 @@ function buildIndex(config) {
         // or reuse it
         fs.mkdirSync(PATH.OBJECT, { recursive: true });
 
-        walkSourceDir([config.vault.root], ctx => {
+        walkSourceDir([vault], ctx => {
             buildObject(index, ctx);
         });
 
-        if (config.vault.cleanup) {
-            fs.readdirSync(config.vault.root).forEach(name => {
-                if (name !== '.gitignore') {
-                    let path = [config.vault.root, name].join('/');
-                    fs.rmSync(path, { recursive: true, force: true });
-                }
-            });
-        }
-
         return index;
     }
 
     let buffer = fs.readFileSync(PATH.INDEX);
     let legacy = decode(buffer);
 
-    walkSourceDir([config.vault.root], ctx => {
+    walkSourceDir([vault], ctx => {
         let path = ctx.path;
         let entry = legacy.object[path];
         // if no such legacy entry found, continue
@@ -197,6 +188,51 @@ function buildIndex(config) {
     return index;
 }
 
+function prepareVault(config) {
+    switch (config["type"]) {
+    case "minio":
+        fs.rmSync(PATH.S3TEMP, { recursive: true, force: true });
+        fs.mkdirSync(PATH.S3TEMP);
+
+        let bucket = config["bucket"];
+        let prefix = config["prefix"];
+        
+        let client = new Client(config);
+        let stream = client.listObjectsV2(bucket, prefix, true);
+
+        stream.on('data', item => {
+            let name = item.name.replace(prefix, '');
+            let path = `${PATH.S3TEMP}/${name}`;
+
+            if (name.endsWith('/')) {
+                if (! fs.existsSync(path)) fs.mkdirSync(path);
+                return;
+            }
+
+            stream.pause();
+            console.log(item.name);
+
+            client.fGetObject(bucket, item.name, path)
+                .catch(err => { throw err })
+                .finally(() => stream.resume());
+        });
+
+        return new Promise((resolve, reject) => {
+            stream.on('close', () => resolve(PATH.S3TEMP));
+            stream.on('error', er => reject(er));
+        });
+    case "file":
+        let path = config["path"];
+
+        if (fs.existsSync(path) && fs.lstatSync(path).isDirectory) {
+            return new Promise((resolve) => resolve(path));
+        }
+        throw new Error(path + ": not a valid vault");
+    default:
+        throw new Error("invalid vault type");
+    }
+}
+
 if (process.argv[1] === import.meta.filename) {
     // when running under node.js solely
     sprachbund().writeBundle();
@@ -213,11 +249,11 @@ export default function sprachbund() {
                 language:    CONFIG?.language,
             });
         },
-        writeBundle() {
+        async writeBundle() {
-            let index = buildIndex(CONFIG);
+            let vault = await prepareVault(CONFIG.vault);
-            let data = encode(index);
+            let index = buildIndex(CONFIG.metadata, vault);
-
+            
-            fs.writeFileSync(PATH.INDEX, data);
+            fs.writeFileSync(PATH.INDEX, encode(index));
         },
     }
 }