Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Initial cleanup of FileSystems and file tree #176

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Initial cleanup of FileSystems and file tree
Lorthiz committed Jun 5, 2024
commit 1dd3647575962a9df8891a714083f67722df2fb4
63 changes: 63 additions & 0 deletions src/v1/FileSystems/FileSystemHelper.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
import {FILE_SYSTEM_TYPE_LOCAL, IFsItem, IFsItemSummary, IJSTreeNodeHelper, ITEM_MODE_DIRECTORY} from "./FileSystemsTypes";


export const findOrCreateFsPath = (root: IFsItem, filePath: string) => {
var currNode = root;
var fnParts = filePath.split("/");
var currPath = "";
for (var i = 0; i < fnParts.length; i++) {
var fnPart = fnParts[i];
currPath += (currPath ? "/" : "") + fnPart;

if (!("children" in currNode)) {
currNode.children = {};
currNode.type = ITEM_MODE_DIRECTORY;
}

if (!(fnPart in currNode.children))
currNode.children[fnPart] = {fsType: root.fsType, type: "file", fn: currPath};

currNode = currNode.children[fnPart];
}
return currNode;
};


export const mapToJSTreeNode = (fsItem: IFsItem, fn: string): IJSTreeNodeHelper => {
const isFolder = fsItem.type === ITEM_MODE_DIRECTORY;
return {
text: fn,
icon: "glyphicon glyphicon-" + (isFolder ? "folder-open" : fn.endsWith(".ksy") ? "list-alt" : "file"),
children: isFolder ? mapToJSTreeNodes(fsItem) : null,
data: fsItem
};
};

export const mapToJSTreeNodes = (fsItem: IFsItem): IJSTreeNodeHelper[] => {
return Object.keys(fsItem.children || [])
.map(k => mapToJSTreeNode(fsItem.children[k], k));
};

export const getSummaryIfPresent = (fsItem?: IFsItem): IFsItemSummary => {
return fsItem
? getSummary(fsItem)
: emptySummary();
};

const getSummary = (data: IFsItem): IFsItemSummary => {
const isFolder = data.type === ITEM_MODE_DIRECTORY;
return {
isFolder: isFolder,
isLocal: data.fsType === FILE_SYSTEM_TYPE_LOCAL,
isKsy: !isFolder && data.fn && data.fn.endsWith(".ksy")
};
};

const emptySummary = (): IFsItemSummary => {
return {
isFolder: false,
isLocal: false,
isKsy: false
};
};

11 changes: 11 additions & 0 deletions src/v1/FileSystems/FileSystemManager.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
import {LocalStorageFileSystem} from "./LocalStorageFileSystem";
import {initKaitaiFs} from "./KaitaiFileSystem";
import {IFileSystemManager} from "./FileSystemsTypes";

const localFs = new LocalStorageFileSystem("fs");
const kaitaiFs = initKaitaiFs();

export var fileSystemsManager: IFileSystemManager = {
local: localFs,
kaitai: kaitaiFs
};
44 changes: 44 additions & 0 deletions src/v1/FileSystems/FileSystemsTypes.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
import {KaitaiFileSystem} from "./KaitaiFileSystem";
import {LocalStorageFileSystem} from "./LocalStorageFileSystem";

export const FILE_SYSTEM_TYPE_LOCAL = "local";
export const FILE_SYSTEM_TYPE_KAITAI = "kaitai";
export const ITEM_MODE_FILE = "file";
export const ITEM_MODE_DIRECTORY = "folder";

export interface IFsItem {
fsType: typeof FILE_SYSTEM_TYPE_LOCAL | typeof FILE_SYSTEM_TYPE_KAITAI;
type: typeof ITEM_MODE_FILE | typeof ITEM_MODE_DIRECTORY;
fn?: string;
children?: { [key: string]: IFsItem; };
}

export interface IFsItemSummary {
isLocal: boolean;
isFolder: boolean;
isKsy: boolean;
}

export interface IFileSystem {
setRootNode(newRoot: IFsItem): Promise<IFsItem>;

getRootNode(): Promise<IFsItem>;

get(fn: string): Promise<string | ArrayBuffer>;

put(fn: string, data: any): Promise<IFsItem>;
}

export interface IFileSystemManager {
local: LocalStorageFileSystem;
kaitai: KaitaiFileSystem;
}

export interface IJSTreeNodeHelper {
id?: string;
text: string;
icon: string;
state?: { opened: boolean; };
children?: IJSTreeNodeHelper[];
data?: IFsItem;
}
83 changes: 83 additions & 0 deletions src/v1/FileSystems/KaitaiFileSystem.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
import {downloadFile} from "../../utils";
import {findOrCreateFsPath, mapToJSTreeNodes} from "./FileSystemHelper";
import {FILE_SYSTEM_TYPE_KAITAI, IFileSystem, IFsItem, IJSTreeNodeHelper} from "./FileSystemsTypes";

declare var kaitaiFsFiles: string[];

export class KaitaiFileSystem implements IFileSystem {
constructor(public files: IFsItem) {
}

getRootNode() {
return Promise.resolve(this.files);
}

setRootNode(newRoot: IFsItem): Promise<IFsItem> {
throw "KaitaiFileSystem.setRootNode is not implemented";
}
get(fn: string): Promise<string | ArrayBuffer> {
if (fn.toLowerCase().endsWith(".ksy"))
return fetch(fn)
.then(response => {
if (!response.ok) {
let msg;
if (response.status === 404) {
msg = "file not found";
} else {
const textAppendix = response.statusText ? ` (${response.statusText})` : "";
msg = `server responded with HTTP status ${response.status}${textAppendix}`;
}
throw new Error(msg);
}
return response.text();
}, err => {
if (err instanceof TypeError) {
throw new Error(`cannot reach the server (message: ${err.message}), check your internet connection`);
}
throw err;
});
else
return downloadFile(fn);
}

put(fn: string, data: any) {
return Promise.reject("KaitaiFileSystem.put is not implemented!");
}
}

export const initKaitaiFsTreeData = (kaitaiFs: KaitaiFileSystem): IJSTreeNodeHelper => {
const root = kaitaiFs.files;

if (!root.children["formats"]) {
console.error("'formats' node is missing from js/kaitaiFsFiles.js, are you sure 'formats' git submodule is initialized? Try run 'git submodule init; git submodule update --recursive; ./genKaitaiFsFiles.py'!");
(<any>root.children["formats"]) = {};
}


return {
text: "kaitai.io",
icon: "glyphicon glyphicon-cloud",
state: {opened: true},
children: [
{
text: "formats",
icon: "glyphicon glyphicon-book",
children: mapToJSTreeNodes(root.children["formats"]),
state: {opened: true}
},
{
text: "samples",
icon: "glyphicon glyphicon-cd",
children: mapToJSTreeNodes(root.children["samples"]),
state: {opened: true}
},
]
};
};

export const initKaitaiFs = () => {
const kaitaiRoot = <IFsItem>{fsType: FILE_SYSTEM_TYPE_KAITAI};
kaitaiFsFiles.forEach(fn => findOrCreateFsPath(kaitaiRoot, fn));
return new KaitaiFileSystem(kaitaiRoot);
};

68 changes: 68 additions & 0 deletions src/v1/FileSystems/LocalStorageFileSystem.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
import {findOrCreateFsPath} from "./FileSystemHelper";
import * as localforage from "localforage";
import {FILE_SYSTEM_TYPE_LOCAL, IFileSystem, IFsItem, IJSTreeNodeHelper, ITEM_MODE_DIRECTORY} from "./FileSystemsTypes";

export class LocalStorageFileSystem implements IFileSystem {
constructor(public prefix: string) {
}

private root: IFsItem;

private filesKey() {
return `${this.prefix}_files`;
}

private fileKey(fn: string) {
return `${this.prefix}_file[${fn}]`;
}

private save() {
return localforage.setItem(this.filesKey(), this.root);
}

async getRootNode() {
if (!this.root)
this.root = await localforage.getItem<IFsItem>(this.filesKey()) ||
<IFsItem>{fsType: FILE_SYSTEM_TYPE_LOCAL, type: ITEM_MODE_DIRECTORY, children: {}};
return this.root;
}

setRootNode(newRoot: IFsItem) {
this.root = newRoot;
return this.save();
}

get(fn: string): Promise<string | ArrayBuffer> {
return localforage.getItem<string | ArrayBuffer>(this.fileKey(fn))
.then(content => {
if (content === null) {
throw new Error("file not found");
}
return content;
});
}

put(fn: string, data: any): Promise<IFsItem> {
return this.getRootNode().then(root => {
const node = findOrCreateFsPath(root, fn);
const saveFileAction = localforage.setItem(this.fileKey(fn), data);
const updateFileTreeAction = this.save();
return Promise.all([saveFileAction, updateFileTreeAction])
.then(_ => node);
});
}
}

export const initLocalStorageFsTreeData = (): IJSTreeNodeHelper => {
return {
text: "Local storage",
id: "localStorage",
icon: "glyphicon glyphicon-hdd",
state: {opened: true},
children: [],
data: {
fsType: FILE_SYSTEM_TYPE_LOCAL,
type: ITEM_MODE_DIRECTORY
}
};
};
7 changes: 4 additions & 3 deletions src/v1/KaitaiServices.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import { fss, IFsItem } from "./app.files";
import { performanceHelper } from "./utils/PerformanceHelper";
import { performanceHelper } from "./utils/PerformanceHelper";
import KaitaiStructCompiler = require("kaitai-struct-compiler");
import * as jsyaml from "js-yaml";
import {fileSystemsManager} from "./FileSystems/FileSystemManager";
import {IFsItem} from "./FileSystems/FileSystemsTypes";

class SchemaUtils {
static ksyNameToJsName(ksyName: string, isProp: boolean) {
@@ -63,7 +64,7 @@ class JsImporter implements IYamlImporter {
const sourceAppendix = mode === 'abs' ? 'kaitai.io' : 'local storage';
let ksyContent;
try {
ksyContent = await fss[importedFsType].get(fn);
ksyContent = await fileSystemsManager[importedFsType].get(fn);
} catch (e) {
const error = new Error(`failed to import spec ${fn} from ${sourceAppendix}${e.message ? ': ' + e.message : ''}`);

275 changes: 54 additions & 221 deletions src/v1/app.files.ts

Large diffs are not rendered by default.

94 changes: 59 additions & 35 deletions src/v1/app.ts
Original file line number Diff line number Diff line change
@@ -1,20 +1,21 @@
import * as localforage from "localforage";
import * as Vue from "vue";

import { UI } from "./app.layout";
import { IFsItem, fss, addKsyFile, refreshFsNodes, initFileTree } from "./app.files";
import { ParsedTreeHandler, IParsedTreeNode } from "./parsedToTree";
import { workerMethods } from "./app.worker";
import { IDataProvider } from "../HexViewer";
import { initFileDrop } from "./FileDrop";
import { performanceHelper } from "./utils/PerformanceHelper";
import { IFileProcessItem, saveFile, precallHook, Delayed } from "../utils";
import { componentLoader } from "../ui/ComponentLoader";
import { ConverterPanelModel } from "../ui/Components/ConverterPanel";
import { exportToJson } from "./ExportToJson";
import {UI} from "./app.layout";
import {addKsyFile, initFileTree, refreshFsNodes} from "./app.files";
import {IParsedTreeNode, ParsedTreeHandler} from "./parsedToTree";
import {workerMethods} from "./app.worker";
import {IDataProvider} from "../HexViewer";
import {initFileDrop} from "./FileDrop";
import {Delayed, IFileProcessItem, saveFile} from "../utils";
import {componentLoader} from "../ui/ComponentLoader";
import {ConverterPanelModel} from "../ui/Components/ConverterPanel";
import {exportToJson} from "./ExportToJson";
import Component from "../ui/Component";
import { CompilerService, CompilationError } from "./KaitaiServices";
import { ErrorWindowHandler } from "./app.errors";
import {CompilationError, CompilerService} from "./KaitaiServices";
import {ErrorWindowHandler} from "./app.errors";
import {fileSystemsManager} from "./FileSystems/FileSystemManager";
import {FILE_SYSTEM_TYPE_KAITAI, IFsItem} from "./FileSystems/FileSystemsTypes";
import KaitaiStructCompiler = require("kaitai-struct-compiler");

$.jstree.defaults.core.force_text = true;
@@ -40,10 +41,21 @@ class AppVM extends Vue {

disableLazyParsing: boolean = false;

public selectInterval(interval: IInterval) { this.selectionChanged(interval.start, interval.end); }
public selectionChanged(start: number, end: number) { this.ui.hexViewer.setSelection(start, end); }
public exportToJson(hex: boolean) { exportToJson(hex).then(json => this.ui.layout.addEditorTab("json export", json, "json")); }
public about() { (<any>$("#welcomeModal")).modal(); }
public selectInterval(interval: IInterval) {
this.selectionChanged(interval.start, interval.end);
}

public selectionChanged(start: number, end: number) {
this.ui.hexViewer.setSelection(start, end);
}

public exportToJson(hex: boolean) {
exportToJson(hex).then(json => this.ui.layout.addEditorTab("json export", json, "json"));
}

public about() {
(<any>$("#welcomeModal")).modal();
}
}

class AppController {
@@ -62,7 +74,10 @@ class AppController {
dataProvider: IDataProvider;
ksyFsItemName = "ksyFsItem";
lastKsyContent: string = null;
isKsyFile(fn: string) { return fn.toLowerCase().endsWith(".ksy"); }

isKsyFile(fn: string) {
return fn.toLowerCase().endsWith(".ksy");
}

compile(srcYamlFsItem: IFsItem, srcYaml: string, kslang: string, debug: true | false | "both"): Promise<any> {
return this.compilerService.compile(srcYamlFsItem, srcYaml, kslang, debug).then(result => {
@@ -78,13 +93,13 @@ class AppController {
var srcYaml = this.ui.ksyEditor.getValue();
var changed = this.lastKsyContent !== srcYaml;

if (changed && (ksyFsItem.fsType === "kaitai" || ksyFsItem.fsType === "static")) {
if (changed && ksyFsItem.fsType === FILE_SYSTEM_TYPE_KAITAI) {
let fsItem = await addKsyFile("localStorage", ksyFsItem.fn.replace(".ksy", "_modified.ksy"), srcYaml);
localforage.setItem(this.ksyFsItemName, fsItem);
}

if (changed)
await fss[ksyFsItem.fsType].put(ksyFsItem.fn, srcYaml);
await fileSystemsManager[ksyFsItem.fsType].put(ksyFsItem.fn, srcYaml);

let compiled = await this.compile(ksyFsItem, srcYaml, "javascript", "both");
if (!compiled) return;
@@ -110,7 +125,7 @@ class AppController {
let jsClassName = this.compilerService.ksySchema.meta.id.split("_").map((x: string) => x.ucFirst()).join("");
await workerMethods.initCode(debugCode, jsClassName, this.compilerService.ksyTypes);

const { result: exportedRoot, error: parseError } = await workerMethods.reparse(this.vm.disableLazyParsing);
const {result: exportedRoot, error: parseError} = await workerMethods.reparse(this.vm.disableLazyParsing);
kaitaiIde.root = exportedRoot;
//console.log("reparse exportedRoot", exportedRoot);

@@ -143,7 +158,7 @@ class AppController {
});

this.errors.handle(parseError);
} catch(error) {
} catch (error) {
this.errors.handle(error);
}
}
@@ -156,7 +171,7 @@ class AppController {
if (!fsItem || fsItem.type !== "file")
return;

var contentRaw = await fss[fsItem.fsType].get(fsItem.fn);
var contentRaw = await fileSystemsManager[fsItem.fsType].get(fsItem.fn);
if (this.isKsyFile(fsItem.fn)) {
let content = <string>contentRaw;
localforage.setItem(this.ksyFsItemName, fsItem);
@@ -193,7 +208,7 @@ class AppController {

addNewFiles(files: IFileProcessItem[]) {
return Promise.all(files.map(file => (this.isKsyFile(file.file.name) ? <Promise<any>>file.read("text") : file.read("arrayBuffer"))
.then(content => fss.local.put(file.file.name, content))))
.then(content => fileSystemsManager.local.put(file.file.name, content))))
.then(fsItems => {
refreshFsNodes();
return fsItems.length === 1 ? this.loadFsItem(fsItems[0]) : Promise.resolve(null);
@@ -207,7 +222,7 @@ class AppController {

onHexViewerSelectionChanged() {
//console.log("setSelection", ui.hexViewer.selectionStart, ui.hexViewer.selectionEnd);
localStorage.setItem("selection", JSON.stringify({ start: this.ui.hexViewer.selectionStart, end: this.ui.hexViewer.selectionEnd }));
localStorage.setItem("selection", JSON.stringify({start: this.ui.hexViewer.selectionStart, end: this.ui.hexViewer.selectionEnd}));

var start = this.ui.hexViewer.selectionStart;
var hasSelection = start !== -1;
@@ -233,6 +248,7 @@ var kaitaiIde = window["kaitaiIde"] = <any>{};
kaitaiIde.version = "0.1";
kaitaiIde.commitId = "";
kaitaiIde.commitDate = "";

//localStorage.setItem("lastVersion", kaitaiIde.version);

interface IInterval {
@@ -244,7 +260,7 @@ $(() => {
$("#webIdeVersion").text(kaitaiIde.version);
$("#webideCommitId")
.attr("href", `https://github.com/kaitai-io/kaitai_struct_webide/commit/${kaitaiIde.commitId}`)
.text(kaitaiIde.commitId.substr(0,7));
.text(kaitaiIde.commitId.substr(0, 7));
$("#webideCommitDate").text(kaitaiIde.commitDate);
$("#compilerVersion").text(KaitaiStructCompiler.version + " (" + KaitaiStructCompiler.buildDate + ")");

@@ -254,7 +270,7 @@ $(() => {

app.init();
componentLoader.load(["Components/ConverterPanel", "Components/Stepper", "Components/SelectionInput"]).then(() => {
new Vue({ data: { model: app.vm.converterPanelModel } }).$mount("#converterPanel");
new Vue({data: {model: app.vm.converterPanelModel}}).$mount("#converterPanel");
app.vm.$mount("#infoPanel");
app.vm.$watch("disableLazyParsing", () => app.reparse());
});
@@ -263,16 +279,24 @@ $(() => {

app.refreshSelectionInput();

app.ui.genCodeDebugViewer.commands.addCommand({ name: "compile", bindKey: { win: "Ctrl-Enter", mac: "Command-Enter" },
exec: function (editor: any) { app.reparse(); } });
app.ui.ksyEditor.commands.addCommand({ name: "compile", bindKey: { win: "Ctrl-Enter", mac: "Command-Enter" },
exec: function (editor: any) { app.recompile(); } });
app.ui.genCodeDebugViewer.commands.addCommand({
name: "compile", bindKey: {win: "Ctrl-Enter", mac: "Command-Enter"},
exec: function (editor: any) {
app.reparse();
}
});
app.ui.ksyEditor.commands.addCommand({
name: "compile", bindKey: {win: "Ctrl-Enter", mac: "Command-Enter"},
exec: function (editor: any) {
app.recompile();
}
});

initFileDrop("fileDrop", (files: any) => app.addNewFiles(files));

async function loadCachedFsItem(cacheKey: string, defFsType: string, defSample: string) {
let fsItem = <IFsItem> await localforage.getItem(cacheKey);
await app.loadFsItem(fsItem || <IFsItem>{ fsType: defFsType, fn: defSample, type: "file" }, false);
let fsItem = <IFsItem>await localforage.getItem(cacheKey);
await app.loadFsItem(fsItem || <IFsItem>{fsType: defFsType, fn: defSample, type: "file"}, false);
}

app.inputReady = loadCachedFsItem("inputFsItem", "kaitai", "samples/sample1.zip");
@@ -294,11 +318,11 @@ $(() => {
$("#hexViewer").on("contextmenu", e => {
downloadInput.toggleClass("disabled", app.ui.hexViewer.selectionStart === -1);

inputContextMenu.css({ display: "block" });
inputContextMenu.css({display: "block"});
var x = Math.min(e.pageX, $(window).width() - inputContextMenu.width());
var h = inputContextMenu.height();
var y = e.pageY > ($(window).height() - h) ? e.pageY - h : e.pageY;
inputContextMenu.css({ left: x, top: y });
inputContextMenu.css({left: x, top: y});
return false;
});