From 7448fe54c582f1c2535c3a41bfb51ac4e00161a6 Mon Sep 17 00:00:00 2001 From: s-ol Date: Mon, 7 Oct 2019 14:17:58 +0000 Subject: SQL import and loading --- .gitignore | 1 + Tupfile | 10 +++---- Tuprules.lua | 13 -------- import_all.moon | 55 ++++++++++++++++++++++++++++++++++ mmm/mmmfs/drivers/sql.moon | 75 ++++++++++++++++++++++------------------------ mmm/mmmfs/drivers/zip.moon | 59 ++++++++++++++++++++++++++++++++++++ mmm/mmmfs/zip.moon | 59 ------------------------------------ render_all.moon | 42 ++++++++++++++++++++++++-- root/Tupdefault.lua | 31 ------------------- spec/driver_spec.moon | 12 ++++---- 10 files changed, 202 insertions(+), 155 deletions(-) delete mode 100644 Tuprules.lua create mode 100644 import_all.moon create mode 100644 mmm/mmmfs/drivers/zip.moon delete mode 100644 mmm/mmmfs/zip.moon delete mode 100644 root/Tupdefault.lua diff --git a/.gitignore b/.gitignore index 34245b3..26b821c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ dist +out ##### TUP GITIGNORE ##### ##### Lines below automatically generated by Tup. ##### Do not edit. diff --git a/Tupfile b/Tupfile index f3e300b..edb8b62 100644 --- a/Tupfile +++ b/Tupfile @@ -4,12 +4,12 @@ include_rules !sassc = |> ^ SCSS %b^ sassc @(SASS_ARGS) %f %o |> | %o.map # download fengari dependencies -: |> !download |> root/fengari-web.js -: |> !download |> root/fengari-web.js.map -: static/* |> cp %f root/ |> root/%b +: |> !download |> out/fengari-web.js +: |> !download |> out/fengari-web.js.map +: static/* |> cp %f out/ |> out/%b # render stylesheet -: scss/main.scss |> !sassc |> root/main.css +: scss/main.scss |> !sassc |> out/main.css # bundle for client loading -: mmm/.bundle.lua | |> ^ WRAP %d^ moon &(build)/bundle_module.moon %o --wrap %f |> root/mmm.bundle.lua +: mmm/.bundle.lua | |> ^ WRAP %d^ moon &(build)/bundle_module.moon %o --wrap %f |> out/mmm.bundle.lua diff --git a/Tuprules.lua b/Tuprules.lua deleted file mode 100644 index cd711a5..0000000 --- a/Tuprules.lua +++ /dev/null @@ -1,13 +0,0 @@ -tup.creategitignore() - -root = tup.nodevariable '.' -build = tup.nodevariable 'build' - -function lua_path() - local LUA_PATH = {} - LUA_PATH += root .. '/?.lua' - LUA_PATH += root .. '/?.server.lua' - LUA_PATH += root .. '/?/init.lua' - LUA_PATH += root .. '/?/init.server.lua' - return 'LUA_PATH="' .. table.concat(LUA_PATH, ';') .. '"' -end diff --git a/import_all.moon b/import_all.moon new file mode 100644 index 0000000..3a2ac61 --- /dev/null +++ b/import_all.moon @@ -0,0 +1,55 @@ +add = (tmpl) -> + package.path ..= ";#{tmpl}.lua" + package.moonpath ..= ";#{tmpl}.moon" + +add '?' +add '?.server' +add '?/init' +add '?/init.server' + +require 'mmm' +require 'lfs' +import Fileder, Key from require 'mmm.mmmfs.fileder' +import SQLStore from require 'mmm.mmmfs.drivers.sql' + +-- usage: +-- moon import_all.moon +{ root } = arg + +assert root, "please specify the root directory" + +-- load a fs file as a fileder facet +load_facet = (filename, filepath) -> + key = (filename\match '(.*)%.%w+') or filename + key = Key key\gsub '%$', '/' + key.filename = filename + + file = assert (io.open filepath, 'r'), "couldn't open facet file '#{filename}'" + value = file\read '*all' + file\close! + + key, value + + +with SQLStore verbose: true + import_fileder = (fileder, dirpath) -> + for file in lfs.dir dirpath + continue if '.' == file\sub 1, 1 + continue if file == 'Tupdefault.lua' + continue if file == 'index.html' + continue if file == '$order' + + filepath = "#{dirpath}/#{file}" + attr = lfs.attributes filepath + switch attr.mode + when 'file' + key, value = load_facet file, filepath + \create_facet fileder, key.name, key.type, value + when 'directory' + next_fileder = \create_fileder fileder, file + -- \create_facet next_fileder, 'name', 'alpha', file + import_fileder next_fileder, filepath + else + warn "unknown entry type '#{attr.mode}'" + + import_fileder '', root diff --git a/mmm/mmmfs/drivers/sql.moon b/mmm/mmmfs/drivers/sql.moon index a961e1e..514985c 100644 --- a/mmm/mmmfs/drivers/sql.moon +++ b/mmm/mmmfs/drivers/sql.moon @@ -1,8 +1,20 @@ sqlite = require 'sqlite3' -class TreeStore - new: (name='db.sqlite3') => - @db = if name then sqlite.open name else sqlite.open_memory! +class SQLStore + new: (opts = {}) => + opts.name or= 'db.sqlite3' + opts.verbose or= false + opts.memory or= false + + if not opts.verbose + @log = -> + + if opts.memory + @log "opening in-memory DB..." + @db = sqlite.open_memory! + else + @log "opening '#{opts.name}'..." + @db = sqlite.open opts.name assert @db\exec [[ PRAGMA foreign_keys = ON; @@ -14,6 +26,8 @@ class TreeStore ON DELETE CASCADE ON UPDATE CASCADE ); + INSERT OR IGNORE INTO fileder (path, parent) VALUES ("", NULL); + CREATE TABLE IF NOT EXISTS facet ( fileder_id INTEGER NOT NULL REFERENCES fileder @@ -28,6 +42,9 @@ class TreeStore CREATE INDEX IF NOT EXISTS facet_name ON facet(name); ]] + log: (...) => + print "[DB]", ... + close: => @db\close! @@ -47,13 +64,13 @@ class TreeStore res = assert stmt\exec! -- fileders - list_fileders_in: (path) => + list_fileders_in: (path='') => coroutine.wrap -> for { path } in @fetch 'SELECT path FROM fileder WHERE parent IS ?', path coroutine.yield path - list_all_fileders: (path) => + list_all_fileders: (path='') => coroutine.wrap -> for path in @list_fileders_in path coroutine.yield path @@ -61,14 +78,19 @@ class TreeStore coroutine.yield p create_fileder: (parent, name) => + path = "#{parent}/#{name}" + + @log "creating fileder #{path}" @exec 'INSERT INTO fileder (path, parent) - VALUES (IFNULL(:parent, "") || "/" || :name, :parent)', - { :parent, :name } + VALUES (:path, :parent)', + { :path, :parent } changes = @fetch_one 'SELECT changes()' assert changes[1] == 1, "couldn't create fileder - parent missing?" + path remove_fileder: (path) => + @log "removing fileder #{path}" @exec 'DELETE FROM fileder WHERE path LIKE :path || "/%" OR path = :path', path @@ -77,7 +99,7 @@ class TreeStore error 'not implemented' @exec 'UPDATE fileder - SET path = IFNULL(parent, "") || "/" || :next_name + SET path = parent || "/" || :next_name WHERE path = :path', { :path, :next_name } @@ -108,6 +130,7 @@ class TreeStore v and v[1] create_facet: (path, name, type, blob) => + @log "creating facet #{path} | #{name}: #{type}" @exec 'INSERT INTO facet (fileder_id, name, type, value) SELECT id, :name, :type, :blob FROM fileder @@ -118,6 +141,7 @@ class TreeStore assert changes[1] == 1, "couldn't create facet - fileder missing?" remove_facet: (path, name, type) => + @log "removing facet #{path} | #{name}: #{type}" @exec 'DELETE FROM facet WHERE name = :name AND type = :type @@ -128,6 +152,7 @@ class TreeStore assert changes[1] == 1, "no such facet" rename_facet: (path, name, type, next_name) => + @log "renaming facet #{path} | #{name}: #{type} -> #{next_name}" @exec 'UPDATE facet SET name = :next_name WHERE name = :name @@ -139,6 +164,7 @@ class TreeStore assert changes[1] == 1, "no such facet" update_facet: (path, name, type, blob) => + @log "updating facet #{path} | #{name}: #{type}" @exec 'UPDATE facet SET value = :blob WHERE facet.name = :name @@ -149,37 +175,6 @@ class TreeStore changes = @fetch_one 'SELECT changes()' assert changes[1] == 1, "no such facet" -load_tree = (file='root.zip') -> - archive = zip.open file - - fileders = setmetatable {}, - __index: (path) => - with val = Fileder {} - .path = path - rawset @, path, val - - fileders['/root'].facets['name: alpha'] = -> 'root' - - for i = 1, #archive - { :name, :size } = archive\stat i - - path, facet = dir_base "/#{name}" - parent, name = dir_base path - - key = load_facet facet - - this = fileders[path] - this.facets['name: alpha'] = -> name - this.facets[key] = -> - file = archive\open i - with file\read size - file\close! - - table_add fileders[parent].children, this - - fileders['/root'] - { - :TreeStore, - load_tree, + :SQLStore } diff --git a/mmm/mmmfs/drivers/zip.moon b/mmm/mmmfs/drivers/zip.moon new file mode 100644 index 0000000..c6417e2 --- /dev/null +++ b/mmm/mmmfs/drivers/zip.moon @@ -0,0 +1,59 @@ +require = relative ..., 1 +import Fileder, Key from require '.fileder' +zip = require 'brimworks.zip' + +-- split filename into dirname + basename +dir_base = (path) -> + dir, base = path\match '(.-)([^/]-)$' + if dir and #dir > 0 + dir = dir\sub 1, #dir - 1 + + dir, base + +-- insert into array unless contained +table_add = (tbl, entry) -> + for _, v in ipairs tbl + if v == entry + return + + table.insert tbl, entry + +-- strip facet formatting +load_facet = (filename) -> + key = (filename\match '(.*)%.%w+') or filename + key = Key key\gsub '%$', '/' + key.filename = filename + + key + +load_tree = (file='root.zip') -> + archive = zip.open file + + fileders = setmetatable {}, + __index: (path) => + with val = Fileder {} + .path = path + rawset @, path, val + + fileders['/root'].facets['name: alpha'] = -> 'root' + + for i = 1, #archive + { :name, :size } = archive\stat i + + path, facet = dir_base "/#{name}" + parent, name = dir_base path + + key = load_facet facet + + this = fileders[path] + this.facets['name: alpha'] = -> name + this.facets[key] = -> + file = archive\open i + with file\read size + file\close! + + table_add fileders[parent].children, this + + fileders['/root'] + +{ :load_tree } diff --git a/mmm/mmmfs/zip.moon b/mmm/mmmfs/zip.moon deleted file mode 100644 index c6417e2..0000000 --- a/mmm/mmmfs/zip.moon +++ /dev/null @@ -1,59 +0,0 @@ -require = relative ..., 1 -import Fileder, Key from require '.fileder' -zip = require 'brimworks.zip' - --- split filename into dirname + basename -dir_base = (path) -> - dir, base = path\match '(.-)([^/]-)$' - if dir and #dir > 0 - dir = dir\sub 1, #dir - 1 - - dir, base - --- insert into array unless contained -table_add = (tbl, entry) -> - for _, v in ipairs tbl - if v == entry - return - - table.insert tbl, entry - --- strip facet formatting -load_facet = (filename) -> - key = (filename\match '(.*)%.%w+') or filename - key = Key key\gsub '%$', '/' - key.filename = filename - - key - -load_tree = (file='root.zip') -> - archive = zip.open file - - fileders = setmetatable {}, - __index: (path) => - with val = Fileder {} - .path = path - rawset @, path, val - - fileders['/root'].facets['name: alpha'] = -> 'root' - - for i = 1, #archive - { :name, :size } = archive\stat i - - path, facet = dir_base "/#{name}" - parent, name = dir_base path - - key = load_facet facet - - this = fileders[path] - this.facets['name: alpha'] = -> name - this.facets[key] = -> - file = archive\open i - with file\read size - file\close! - - table_add fileders[parent].children, this - - fileders['/root'] - -{ :load_tree } diff --git a/render_all.moon b/render_all.moon index 3d295c9..e2f171e 100644 --- a/render_all.moon +++ b/render_all.moon @@ -10,7 +10,6 @@ add '?/init.server' require 'mmm' import tohtml from require 'mmm.component' import Browser from require 'mmm.mmmfs.browser' -import load_tree from require 'mmm.mmmfs.zip' import get_meta, header, footer from require 'build.layout' export BROWSER @@ -70,7 +69,46 @@ render = (fileder, output) -> " \close! -for fileder in coroutine.wrap load_tree!\iterate +import SQLStore from require 'mmm.mmmfs.drivers.sql' +import Fileder, Key from require 'mmm.mmmfs.fileder' + +-- split filename into dirname + basename +dir_base = (path) -> + dir, base = path\match '(.-)([^/]-)$' + if dir and #dir > 0 + dir = dir\sub 1, #dir - 1 + + dir, base + +load_tree = (store, root='') -> + fileders = setmetatable {}, + __index: (path) => + with val = Fileder {} + .path = path + rawset @, path, val + + root = fileders[root] + root.facets['name: alpha'] = '' + for fn, ft in store\list_facets root.path + val = store\load_facet root.path, fn, ft + root.facets[Key fn, ft] = val + + for path in store\list_all_fileders root.path + fileder = fileders[path] + + parent, name = dir_base path + fileder.facets['name: alpha'] = name + table.insert fileders[parent].children, fileder + + for fn, ft in store\list_facets path + val = store\load_facet path, fn, ft + fileder.facets[Key fn, ft] = val + + root + +tree = load_tree SQLStore! + +for fileder in coroutine.wrap tree\iterate print "rendering '#{fileder.path}'..." os.execute "mkdir -p 'out/#{fileder.path}'" render fileder, "out/#{fileder.path}/index.html" diff --git a/root/Tupdefault.lua b/root/Tupdefault.lua deleted file mode 100644 index 260ca2e..0000000 --- a/root/Tupdefault.lua +++ /dev/null @@ -1,31 +0,0 @@ -local LUA_PATH = lua_path() -bundle = LUA_PATH .. ' moon ' .. build .. '/bundle_fileder.moon' -render = LUA_PATH .. ' moon ' .. build .. '/render_fileder.moon' - -local _facets, facets = {}, {} -for i, file in ipairs(tup.glob '*$*') do _facets[file] = true end -for i, file in ipairs(tup.glob '*:*') do _facets[file] = true end -for i, file in ipairs(tup.glob '*->*') do _facets[file] = true end - -for file in pairs(_facets) do facets += file end -table.sort(facets) - -local inputs = '' -for i, file in ipairs(facets) do - inputs = inputs .. " '" .. file .. "'" -end - -facets += '' -facets += root .. '/' - -tup.rule( - facets, - '^ BNDL %d^ ' .. bundle .. ' ' .. root .. ' %d ' .. inputs .. ' -- %', - { '$bundle.lua', '../' } -) - -tup.rule( - '$bundle.lua', - '^ HTML %d^ ' .. render .. ' ' .. root, - 'index.html' -) diff --git a/spec/driver_spec.moon b/spec/driver_spec.moon index 60cd574..379dcac 100644 --- a/spec/driver_spec.moon +++ b/spec/driver_spec.moon @@ -1,4 +1,4 @@ -import TreeStore from require 'mmm.mmmfs.drivers.sql' +import SQLStore from require 'mmm.mmmfs.drivers.sql' sort2 = (a, b) -> {ax, ay}, {bx, by} = a, b @@ -11,7 +11,7 @@ toseq2 = (iter) -> with v = [{x, y} for x, y in iter] describe "sql driver", -> randomize false - ts = TreeStore! + ts = SQLStore memory: true it "starts out empty", -> assert.are.same {}, toseq ts\list_fileders_in! @@ -22,12 +22,14 @@ describe "sql driver", -> ts\create_fileder '/hello', 'world' it "can create root fileders", -> - ts\create_fileder nil, 'hello' + assert.are.same '/hello', ts\create_fileder '', 'hello' assert.are.same {'/hello'}, toseq ts\list_all_fileders! it "can create and list child fileders recursively", -> - ts\create_fileder '/hello', 'world' - ts\create_fileder '/hello/world', 'again' + assert.are.same '/hello/world', + ts\create_fileder '/hello', 'world' + assert.are.same '/hello/world/again', + ts\create_fileder '/hello/world', 'again' assert.are.same {'/hello', '/hello/world', '/hello/world/again'}, toseq ts\list_all_fileders! -- cgit v1.2.3