This is an automated email from the ASF dual-hosted git repository. SpriCoder pushed a commit to branch fs/inner-view in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 0065d16a0c6caec5f340b760a1df6b4e8d0bac22 Author: spricoder <[email protected]> AuthorDate: Thu Apr 30 19:03:40 2026 +0800 check old version --- .../plans/2026-04-29-cli-filesystem-mode.md | 41 +++-- .../specs/2026-04-29-cli-filesystem-mode-design.md | 70 ++++---- .../org/apache/iotdb/cli/fs/FilesystemShell.java | 68 ++++++- .../org/apache/iotdb/cli/fs/node/FsNodeType.java | 2 - .../cli/fs/provider/TableCsvAppendPlanner.java | 30 +--- .../provider/TableFilesystemMutationProvider.java | 21 ++- .../fs/provider/TableFilesystemSchemaProvider.java | 174 +++++++----------- .../TableFilesystemSql.java} | 31 ++-- .../apache/iotdb/cli/CliFilesystemModeTest.java | 9 +- .../apache/iotdb/cli/fs/FilesystemShellTest.java | 197 ++++++++++++++------- .../fs/command/FilesystemCommandParserTest.java | 59 +++--- .../TableFilesystemMutationProviderTest.java | 23 ++- .../TableFilesystemSchemaProviderTest.java | 190 ++++++++++++-------- 13 files changed, 526 insertions(+), 389 deletions(-) diff --git a/docs/superpowers/plans/2026-04-29-cli-filesystem-mode.md b/docs/superpowers/plans/2026-04-29-cli-filesystem-mode.md index 0a85d3525b5..a39d65c880c 100644 --- a/docs/superpowers/plans/2026-04-29-cli-filesystem-mode.md +++ b/docs/superpowers/plans/2026-04-29-cli-filesystem-mode.md @@ -73,8 +73,8 @@ These notes capture follow-up implementation experience for quickly resuming thi - `ls -a` and `ll -a` include `.` and `..`; `ll` reuses `ls` option parsing as the long-listing alias. - `tree` prints indented names only. - - `cat /db/table.csv` prints CSV records. Legacy compatibility table/column paths and `paste` - continue to print tab-separated values. + - `cat /db/table.csv`, `cat /db/table.schema`, and `cat /db/table.meta` print regular file + content as CSV lines. - `cut -d, -f2,3 /db/table.csv` is the Unix-compatible multi-field projection form for CSV-first table files. It is delimiter-based text cutting, not CSV quote parsing and not a database column-selection dialect. @@ -82,8 +82,8 @@ These notes capture follow-up implementation experience for quickly resuming thi default read limit. - `stat` is the place to show metadata. - Do not add filesystem command dialects such as `cat --columns` or `select`. Multi-column reads - should use `cut` for CSV-first table files; legacy column paths may still use - `paste /db/table/col1 /db/table/col2`. + over table data should use `cut` for CSV-first table files. `paste` remains the Unix command for + side-by-side line joining of regular files, not a database column-selection dialect. - Table provider can optimize Unix-looking commands internally: - Table mode is CSV-first: a database is a directory, and each table is exposed as `/db/table.csv` with `/db/table.schema` and `/db/table.meta` sidecar regular files. @@ -94,9 +94,10 @@ These notes capture follow-up implementation experience for quickly resuming thi preserved. - `cat /db/table.meta` -> `SHOW TABLES DETAILS FROM db`, filtered to the table and formatted as CSV with IoTDB result columns preserved. - - Legacy `/db/table/column` paths may remain as compatibility paths or migration sources. - - Legacy `cat /db/table/col` -> `SELECT col FROM db.table LIMIT 20` - - Legacy `paste /db/table/col1 /db/table/col2` -> `SELECT col1, col2 FROM db.table LIMIT 20` + - Bare table paths such as `/db/table` and column paths such as `/db/table/column` are not part + of the sidecar model and should resolve to `unknown` or a not-readable error. + - `paste /db/table1.csv /db/table2.csv` -> read each regular file as lines and join + corresponding lines with tabs, padding missing trailing lines with empty fields. - Table-mode write boundaries are intentionally narrow and only active with `--fs_write_mode enabled`: - `mkdir /db` creates a database. @@ -113,10 +114,10 @@ These notes capture follow-up implementation experience for quickly resuming thi - Interactive filesystem command errors must be handled at the single-command loop level. A `SQLException` from `FilesystemShell.execute()` should print `<command>: <message>` and continue the prompt, not bubble out to `receiveCommands()` and exit the CLI. -- The observed `cat time` exit came from path resolution and error bubbling: from `/testtest`, - `cat time` resolves to `/testtest/time`; table mode interpreted that as table `testtest.time`; - the server returned `550`; the unchecked propagation exited the CLI. Keep a regression test for - this behavior. +- The observed `cat time` exit came from path resolution and error bubbling in the older table-path + model. In the sidecar model, from `/testtest`, `cat time` resolves to `/testtest/time`, which is + not a sidecar file and should report `cat: Path is not readable: /testtest/time` while keeping the + prompt alive. ## Data Append Design @@ -131,7 +132,8 @@ Design decisions already approved: - Only append writes are supported. Truncate, overwrite, random writes, row deletion, and operating system redirection such as `>> /db/table.csv` are out of scope. - The target must be a table-mode data file `/<database>/<table>.csv`. -- `tee` without `-a`, sidecar files, legacy column paths, and tree-mode paths are rejected. +- `tee` without `-a`, sidecar files, bare table paths, column paths, and tree-mode paths are + rejected. - Writes require `--access_mode filesystem --sql_dialect table --fs_write_mode enabled`. - Non-interactive mode reads CSV from stdin until EOF and then submits. - Interactive mode enters an append buffer. `:wq` validates and submits, `:q!` discards, and `:q` @@ -166,16 +168,16 @@ Design decisions already approved: | `stat [path]` | Print filesystem-style metadata for a node. | `stat /db/table.csv` | | `cat <path>...` | Print readable paths sequentially. | `cat /db/table.csv` | | `head [-n lines] <path>` | Print the first rows or text lines; short form such as `-5` is accepted. | `head -n 5 /db/table.csv` | -| `tail [-n lines] <path>` | Print the last rows or text lines where supported. | `tail -n 5 /db/table.csv` | -| `wc -l <path>` | Print logical count and path. | `wc -l /db/table.csv` | +| `tail [-n lines] <path>` | Print the last file lines where supported. | `tail -n 5 /db/table.csv` | +| `wc -l <path>` | Print file-line count and path. | `wc -l /db/table.schema` | | `grep <pattern> <path>` | Print rows or lines containing a literal substring. | `grep spricoder /db/table.csv` | | `find [path] [-name name]` | Recursively print matching paths; `-name` is exact node-name matching. | `find /db -name table.csv` | | `less <path>` | Non-interactive read alias using the default read limit. | `less /db/table.csv` | | `more <path>` | Non-interactive read alias using the default read limit. | `more /db/table.schema` | | `file <path>` | Print `directory`, `regular file`, or `unknown`. | `file /db/table.meta` | -| `du <path>` | Print logical count and path using provider count. | `du /db/table.csv` | +| `du <path>` | Print provider count and path; table sidecars use file-line counts. | `du /db/table.csv` | | `cut -d<delimiter> -f<fields> <path>` | Delimiter-based Unix field selection; supports lists and closed ranges. | `cut -d, -f2,3 /db/table.csv` | -| `paste <path>...` | Print multiple readable paths side by side; table mode supports legacy same-table column paths. | `paste /db/table/key /db/table/value` | +| `paste <path>...` | Print multiple regular files side by side, joining corresponding lines with tabs. | `paste /db/t1.csv /db/t2.csv` | | `tree [-L depth] [path]` | Print descendants with indentation and names only. | `tree -L 2 /db` | | `mkdir <path>` | Write-gated; in table mode with writes enabled, creates a database. | `mkdir /newdb` | | `rm <path>` | Write-gated; in table mode with writes enabled, only table CSV drop is allowed. | `rm /db/table.csv` | @@ -360,7 +362,8 @@ Use a mocked `SqlExecutor` to verify `list(/)`, `list(/db)`, `describe(/db/table `read(/db/table.csv)`, `describe(/db/table.schema)`, `readLines(/db/table.schema)`, `describe(/db/table.meta)`, and `readLines(/db/table.meta)` issue expected SQL and return typed nodes or text lines. -Also cover legacy `/db/table/col` paths if compatibility behavior remains enabled. +Also cover that bare table paths such as `/db/table` and column paths such as `/db/table/col` are +not readable filesystem objects in the sidecar model. - [ ] **Step 2: Run test to verify it fails** @@ -371,8 +374,8 @@ Expected: compilation failure or missing behavior. Implement CSV-first database/table sidecar mapping with centralized identifier rendering: `/db/table.csv` is the table data regular file, `/db/table.schema` is the schema sidecar regular -file, and `/db/table.meta` is the metadata sidecar regular file. Keep `/db/table/column` only as a -legacy compatibility path or migration source. +file, and `/db/table.meta` is the metadata sidecar regular file. Do not expose `/db/table` as a +directory or `/db/table/column` as a regular file. - [ ] **Step 4: Run test to verify it passes** diff --git a/docs/superpowers/specs/2026-04-29-cli-filesystem-mode-design.md b/docs/superpowers/specs/2026-04-29-cli-filesystem-mode-design.md index baeee519fd2..ed1d86742fd 100644 --- a/docs/superpowers/specs/2026-04-29-cli-filesystem-mode-design.md +++ b/docs/superpowers/specs/2026-04-29-cli-filesystem-mode-design.md @@ -155,9 +155,10 @@ The table data file is the primary path for user data operations: - `/<database>/<table>.meta`: sidecar table metadata rendered as CSV from the table row returned by `SHOW TABLES DETAILS FROM <database>`, preserving the column names and values returned by IoTDB. -The legacy column-oriented path `/<database>/<table>/<column>` is not the primary table-model -filesystem abstraction. It may remain as a compatibility path for existing read behavior during a -migration period, or be documented as the migration source when moving users to the sidecar model. +Bare table paths and column-oriented paths are intentionally outside the table filesystem model. +`/<database>/<table>` must not be exposed as a directory, and +`/<database>/<table>/<column>` must not be exposed as a regular file. Users inspect the table +through the `.csv`, `.schema`, and `.meta` sidecars. | Filesystem Path | IoTDB Object | Node Type | Discovery | | --- | --- | --- | --- | @@ -166,11 +167,11 @@ migration period, or be documented as the migration source when moving users to | `/<database>/<table>.csv` | Table data regular file | `TABLE_DATA_FILE` | `SHOW TABLES FROM <database>` | | `/<database>/<table>.schema` | Schema sidecar regular file | `TABLE_SCHEMA_FILE` | Exists via `SHOW TABLES FROM <database>`; content via `DESC <database>.<table> DETAILS` | | `/<database>/<table>.meta` | Metadata sidecar regular file | `TABLE_META_FILE` | Exists via `SHOW TABLES FROM <database>`; content via `SHOW TABLES DETAILS FROM <database>` | -| `/<database>/<table>/<column>` | Legacy column compatibility path | `TABLE_COLUMN` | `DESC <database>.<table> DETAILS` | Table-model devices from `SHOW DEVICES FROM <table>` are not part of the first version's base path -hierarchy because they are data-instance-oriented rather than schema-container-oriented. They can -be added later as a virtual directory such as `/<database>/<table>/.devices`. +hierarchy because they are data-instance-oriented rather than schema-container-oriented. If device +metadata is exposed later, it must use a sidecar-compatible regular file path rather than turning +`/<database>/<table>` into a directory. ## Path Rules @@ -182,8 +183,8 @@ be added later as a virtual directory such as `/<database>/<table>/.devices`. - Table-model paths use `/database/table.csv`, `/database/table.schema`, and `/database/table.meta`. The database component is a directory; the table data and sidecar paths are regular files. -- Legacy table-model paths of the form `/database/table/column` are compatibility paths or - migration inputs, not the primary table filesystem model. +- Table-model paths of the form `/database/table` and `/database/table/column` are not filesystem + objects in the sidecar model. - Wildcard paths are not treated as filesystem nodes in the first version. Users should run wildcard SQL through normal SQL mode. - SQL escaping and identifier quoting must be centralized in provider helper methods. Command @@ -203,18 +204,18 @@ semantics wherever the same command exists. Provider support can still vary by d | `ll [-a] [path]` | Long listing alias. Uses read-only permissions by default: directories as `dr-xr-xr-x`, files as `-r--r--r--`. | `ll -a /db` | | `cd <path>` | Change the current directory only if the target is a directory node. | `cd /db` | | `stat [path]` | Print filesystem-style metadata, including path, Unix type, and provider metadata. | `stat /db/table.csv` | -| `cat <path>...` | Print one or more readable paths sequentially. Table `.csv`, `.schema`, and `.meta` sidecars print CSV lines; legacy table/column paths print tab-separated values. | `cat /db/table.csv` | +| `cat <path>...` | Print one or more readable paths sequentially. Table `.csv`, `.schema`, and `.meta` sidecars print CSV lines. | `cat /db/table.csv` | | `head [-n lines] <path>` | Print the first rows or text lines for a readable path. Short numeric form such as `head -5 <path>` is also parsed. | `head -n 5 /db/table.csv` | -| `tail [-n lines] <path>` | Print the last rows or text lines where the provider supports tail. Table `.csv` uses `ORDER BY time DESC LIMIT n` internally and returns original order. | `tail -n 5 /db/table.csv` | -| `wc -l <path>` | Print logical row or line count plus path. Only `-l` is supported. | `wc -l /db/table.csv` | +| `tail [-n lines] <path>` | Print the last file lines where the provider supports tail. Table `.csv` uses `ORDER BY time DESC LIMIT n` internally and returns original order. | `tail -n 5 /db/table.csv` | +| `wc -l <path>` | Print file-line count plus path. Only `-l` is supported. | `wc -l /db/table.schema` | | `grep <pattern> <path>` | Print lines or rows containing the literal pattern. This is substring matching, not regular-expression matching. | `grep spricoder /db/table.csv` | | `find [path] [-name name]` | Recursively list the starting path and descendants whose node name exactly matches `name`; without `-name`, it prints all visited paths. | `find /db -name table.csv` | | `less <path>` | Current implementation prints readable content like `cat` with the default read limit; it is not an interactive pager. | `less /db/table.csv` | | `more <path>` | Current implementation prints readable content like `cat` with the default read limit; it is not an interactive pager. | `more /db/table.schema` | | `file <path>` | Print the Unix type for the path: `directory`, `regular file`, or `unknown`. | `file /db/table.meta` | -| `du <path>` | Print logical count plus path, using the provider count operation. | `du /db/table.csv` | +| `du <path>` | Print provider count plus path. Table sidecars use file-line counts. | `du /db/table.csv` | | `cut -d<delimiter> -f<fields> <path>` | Apply Unix delimiter-based field selection to each line. The delimiter must be one character. Field lists and closed ranges such as `2,3` and `1-2` are supported. | `cut -d, -f2,3 /db/table.csv` | -| `paste <path>...` | Read multiple file-like paths side by side. Table mode currently supports legacy same-table column paths and optimizes them to one SQL projection. | `paste /db/table/key /db/table/value` | +| `paste <path>...` | Read multiple regular files side by side and join corresponding lines with tabs. | `paste /db/t1.csv /db/t2.csv` | | `tree [-L depth] [path]` | Recursively print descendants with indentation and names only. `-L` limits recursion depth. | `tree -L 2 /db` | | `mkdir <path>` | Write-gated command. With table mode and `--fs_write_mode enabled`, `mkdir /db` creates a database. Otherwise it returns a read-only or unsupported error. | `mkdir /newdb` | | `rm <path>` | Write-gated command. With table mode and `--fs_write_mode enabled`, only `rm /db/table.csv` is allowed and maps to table drop. | `rm /db/table.csv` | @@ -252,11 +253,12 @@ Raw SQL should be run in the default SQL access mode. | `stat /db/table.meta` | `SHOW TABLES FROM db`, filtered to the table and rendered as filesystem metadata for the metadata sidecar. | | `cat /db/table.csv` | `SELECT * FROM db.table LIMIT <limit>`, formatted as CSV records. | | `cut -d, -f2,3 /db/table.csv` | Delimiter-based text field projection over the CSV records. | +| `paste /db/t1.csv /db/t2.csv` | Read each regular file as CSV/text lines and join corresponding lines with tabs. | | `tee -a /db/table.csv` | Parse CSV input with Apache Commons CSV, validate columns and required `time`, then execute chunked `INSERT INTO db.table(...) VALUES ...`. | | `cat /db/table.schema` | `DESC db.table DETAILS`, formatted as CSV with IoTDB result columns preserved. | | `cat /db/table.meta` | `SHOW TABLES DETAILS FROM db`, filtered to the table and formatted as CSV with IoTDB result columns preserved. | -| `stat /db/table/col` | Legacy compatibility: `DESC db.table DETAILS`, filtered to the column. | -| `cat /db/table/col` | Legacy compatibility: `SELECT col FROM db.table LIMIT <limit>`. | +| `/db/table` | Not a table-mode filesystem object in the sidecar model; should describe as `unknown` and reject reads. | +| `/db/table/col` | Not a table-mode filesystem object in the sidecar model; should describe as `unknown` and reject reads. | ## Unix Output Semantics @@ -267,12 +269,12 @@ exposing internal implementation types or Java debug-style structures in normal `ls -1`; it must not introduce comma-separated output or database-specific listing dialects. - `ls -a` and `ll -a` include `.` and `..` before normal entries. - `tree` prints the hierarchy with indentation and names only. -- `cat` prints regular file content without Java object formatting. For table data files this is - CSV; for legacy compatibility table/column paths this remains tab-separated row values. +- `cat` prints regular file content without Java object formatting. For table sidecars this is CSV. - `cut -d, -f2,3 /db/table.csv` is the Unix-compatible way to project fields from table CSV content. It performs delimiter-based text cutting like Unix `cut`; it does not parse CSV quoting or introduce table-specific column-selection flags. -- `paste` prints multiple file-like paths side by side as tab-separated values. +- `paste` prints multiple regular files side by side as tab-separated values. It does not select + database columns. - `less` and `more` are currently non-interactive read aliases with the default read limit. - `stat` is the command that may expose typed metadata, because Unix `stat` is explicitly about object metadata. @@ -282,9 +284,9 @@ exposing internal implementation types or Java debug-style structures in normal - Error output should follow a command-prefixed style such as `cat: <message>` or `cd: <path>: Not a directory`. -This means provider-internal node types such as `TABLE_DATABASE`, `TABLE_COLUMN`, or +This means provider-internal node types such as `TABLE_DATABASE`, `TABLE_DATA_FILE`, or `TREE_DATABASE` must not appear in `ls` or `tree` output. Similarly, `SqlRow.asMap().toString()` -must not be used for `cat` or `paste` output. +must not be used for `cat`, `paste`, or sidecar output. ## Completion Semantics @@ -305,23 +307,21 @@ Table-mode read behavior is currently: - `cat /db/table.schema` maps to `DESC db.table DETAILS` and preserves IoTDB result columns in CSV. - `cat /db/table.meta` maps to `SHOW TABLES DETAILS FROM db`, filters to the table row, and preserves IoTDB result columns in CSV. -- Legacy `cat /db/table/column` may map to `SELECT column FROM db.table LIMIT <limit>` during the - migration period. -- Legacy `paste /db/table/col1 /db/table/col2` may map to - `SELECT col1, col2 FROM db.table LIMIT <limit>` when all paths are columns from the same table. +Table-mode no longer exposes `/db/table` as a directory or `/db/table/column` as a file. Those paths +are unknown in the sidecar model and must not trigger table or column SQL reads. -Although `paste` is implemented through one optimized SQL query for table mode, the user-facing -semantics remain Unix-like: users pass multiple file paths to a standard Unix command rather than -using a database-specific `select` command or `cat --columns` dialect. +`paste` remains Unix-like: users pass multiple regular file paths and the shell joins +corresponding lines with tabs. It must not become a database-specific `select` command or +`cat --columns` dialect. For CSV-first table files, multi-column projection should prefer Unix `cut` syntax such as `cut -d, -f2,3 /db/table.csv`. The implementation may later optimize this internally, but the public interface must remain the standard `cut` form. In interactive filesystem mode, a single command failure must not terminate the CLI session. For -example, if `cat time` is resolved from `/testtest` to `/testtest/time`, table mode treats that as a -table path and may receive a server error such as `550: Table 'testtest.time' does not exist`. That -error should be printed as `cat: 550: ...`, then the prompt should continue. +example, if `cat time` is resolved from `/testtest` to `/testtest/time`, table mode should reject it +as not readable because it is not `.csv`, `.schema`, or `.meta`. The error should be printed with +the command name, then the prompt should continue. ## Append Data Write Semantics @@ -347,7 +347,7 @@ The sidecar files remain read-only metadata views: - `tee -a /db/table.schema` is rejected. - `tee -a /db/table.meta` is rejected. -- Legacy column paths such as `/db/table/col` are not writable append targets. +- Column paths such as `/db/table/col` are not filesystem objects and are not writable append targets. - Tree-mode paths remain non-writable even when `--fs_write_mode enabled` is set. ### Input Modes @@ -454,8 +454,8 @@ Filesystem mode separates reads from mutations. The schema provider owns read op - `readLines(FsPath path)` for text sidecars such as `.csv`, `.schema`, and `.meta` - `tail(FsPath path)` / `tailLines(FsPath path)` where the provider supports tail - `count(FsPath path)` where the provider supports logical count -- `read(List<FsPath> paths)` for provider-optimized multi-path reads such as legacy table-column - `paste` +- `read(List<FsPath> paths)` remains available for providers that need optimized multi-path reads, + but table mode sidecar `paste` is implemented as regular file line joining in the shell. The mutation provider owns the current write-gated operations: @@ -541,8 +541,8 @@ Unit tests should cover the behavior without needing a live IoTDB instance where commands, and the parser-only `sql` form. - Provider tests with a mocked `SqlExecutor`, verifying tree-mode path-to-SQL mapping. - Provider tests with a mocked `SqlExecutor`, verifying table-mode path-to-SQL mapping, CSV - sidecars, IoTDB-preserved `.schema`/`.meta` content, multi-column legacy reads, and table - mutation restrictions. + sidecars, IoTDB-preserved `.schema`/`.meta` content, rejection of bare table and column paths, + and table mutation restrictions. - CLI option tests extending existing CLI unit coverage for default `access_mode`, filesystem mode, invalid mode values, and `fs_write_mode`. - Shell tests proving SQL mode remains the default and filesystem mode dispatches to diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/fs/FilesystemShell.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/fs/FilesystemShell.java index 63c543d40c6..051c96b5b35 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/fs/FilesystemShell.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/fs/FilesystemShell.java @@ -84,10 +84,10 @@ public class FilesystemShell { ctx.getPrinter().println(currentPath.toString()); return true; case LS: - printNodes(provider.list(resolve(command.getPath())), isAllOption(command)); + printList(command.getPath(), isAllOption(command), false); return true; case LL: - printLongNodes(provider.list(resolve(command.getPath())), isAllOption(command)); + printList(command.getPath(), isAllOption(command), true); return true; case CD: changeDirectory(command.getPath()); @@ -136,7 +136,7 @@ public class FilesystemShell { printCut(command.getPath(), command.getOption(), command.getPattern()); return true; case PASTE: - printRows(provider.read(resolve(command.getPaths()), DEFAULT_READ_LIMIT)); + printPaste(command.getPaths()); return true; case TEE: append(command.getPath(), false); @@ -173,6 +173,15 @@ public class FilesystemShell { } private void printTree(FsPath path, int depth) throws SQLException { + FsNode node = provider.describe(path); + if (node.getType() == FsNodeType.UNKNOWN) { + ctx.getPrinter().println("tree: " + path + ": No such file or directory"); + return; + } + if (!isDirectory(node.getType())) { + ctx.getPrinter().println(node.getName()); + return; + } printTreeChildren(path, 0, depth); } @@ -218,6 +227,28 @@ public class FilesystemShell { return resolvedPaths; } + private void printList(String path, boolean all, boolean longListing) throws SQLException { + FsPath resolvedPath = resolve(path); + FsNode node = provider.describe(resolvedPath); + if (node.getType() == FsNodeType.UNKNOWN) { + ctx.getPrinter().println("ls: " + resolvedPath + ": No such file or directory"); + return; + } + if (!isDirectory(node.getType())) { + if (longListing) { + ctx.getPrinter().println(longMode(node.getType()) + " 1 iotdb iotdb 0 " + node.getName()); + } else { + ctx.getPrinter().println(node.getName()); + } + return; + } + if (longListing) { + printLongNodes(provider.list(resolvedPath), all); + } else { + printNodes(provider.list(resolvedPath), all); + } + } + private void printNodes(List<FsNode> nodes, boolean all) { if (all) { ctx.getPrinter().println("."); @@ -316,6 +347,19 @@ public class FilesystemShell { } } + private void printPaste(List<String> paths) throws SQLException { + List<List<String>> files = new ArrayList<>(); + int maxLines = 0; + for (String path : paths) { + List<String> lines = readableLines(resolve(path), DEFAULT_READ_LIMIT); + files.add(lines); + maxLines = Math.max(maxLines, lines.size()); + } + for (int i = 0; i < maxLines; i++) { + ctx.getPrinter().println(pasteLine(files, i)); + } + } + private List<String> readableLines(FsPath path, int limit) throws SQLException { if (isTextFile(path)) { return provider.readLines(path, limit); @@ -495,9 +539,7 @@ public class FilesystemShell { || type == FsNodeType.TREE_DATABASE || type == FsNodeType.TREE_INTERNAL_PATH || type == FsNodeType.TREE_DEVICE - || type == FsNodeType.TABLE_DATABASE - || type == FsNodeType.TABLE_TABLE - || type == FsNodeType.TABLE_VIEW; + || type == FsNodeType.TABLE_DATABASE; } private static String longMode(FsNodeType type) { @@ -540,6 +582,20 @@ public class FilesystemShell { return builder.toString(); } + private static String pasteLine(List<List<String>> files, int lineIndex) { + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < files.size(); i++) { + if (i > 0) { + builder.append('\t'); + } + List<String> lines = files.get(i); + if (lineIndex < lines.size()) { + builder.append(lines.get(lineIndex)); + } + } + return builder.toString(); + } + private static boolean[] selectedFields(String fields, int fieldCount) { boolean[] selected = new boolean[fieldCount]; for (String field : fields.split(",")) { diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/fs/node/FsNodeType.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/fs/node/FsNodeType.java index df63ef5c817..0d7fb818d47 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/fs/node/FsNodeType.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/fs/node/FsNodeType.java @@ -27,9 +27,7 @@ public enum FsNodeType { TREE_DEVICE, TREE_TIMESERIES, TABLE_DATABASE, - TABLE_TABLE, TABLE_VIEW, - TABLE_COLUMN, TABLE_DATA_FILE, TABLE_SCHEMA_FILE, TABLE_META_FILE, diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/fs/provider/TableCsvAppendPlanner.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/fs/provider/TableCsvAppendPlanner.java index f5a0150c8e0..5ac20fa9013 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/fs/provider/TableCsvAppendPlanner.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/fs/provider/TableCsvAppendPlanner.java @@ -46,7 +46,8 @@ class TableCsvAppendPlanner { private TableCsvAppendPlanner() {} - static List<String> plan(String tablePath, List<SqlRow> schemaRows, List<String> lines) + static List<String> plan( + String database, String table, List<SqlRow> schemaRows, List<String> lines) throws SQLException { List<CSVRecord> records = parse(lines); if (records.isEmpty()) { @@ -60,7 +61,7 @@ class TableCsvAppendPlanner { if (csv.rows.isEmpty()) { return new ArrayList<>(); } - return buildStatements(tablePath, csv.columns, csv.rows); + return buildStatements(database, table, csv.columns, csv.rows); } private static List<CSVRecord> parse(List<String> lines) throws SQLException { @@ -213,13 +214,13 @@ class TableCsvAppendPlanner { } private static List<String> buildStatements( - String tablePath, List<TableColumn> columns, List<String> rows) { + String database, String table, List<TableColumn> columns, List<String> rows) { List<String> statements = new ArrayList<>(); for (int start = 0; start < rows.size(); start += INSERT_BATCH_SIZE) { int end = Math.min(start + INSERT_BATCH_SIZE, rows.size()); statements.add( "INSERT INTO " - + identifierPath(tablePath) + + TableFilesystemSql.tablePath(database, table) + "(" + columnList(columns) + ") VALUES " @@ -228,33 +229,14 @@ class TableCsvAppendPlanner { return statements; } - private static String identifierPath(String tablePath) { - String[] parts = tablePath.split("\\.", -1); - StringBuilder builder = new StringBuilder(); - for (String part : parts) { - if (builder.length() > 0) { - builder.append('.'); - } - builder.append(identifier(part)); - } - return builder.toString(); - } - private static String columnList(List<TableColumn> columns) { List<String> names = new ArrayList<>(); for (TableColumn column : columns) { - names.add(identifier(column.name)); + names.add(TableFilesystemSql.identifier(column.name)); } return join(names, ", "); } - private static String identifier(String value) { - if (value.matches("[A-Za-z_][A-Za-z0-9_]*")) { - return value; - } - return "\"" + value.replace("\"", "\"\"") + "\""; - } - private static String join(List<String> values, String delimiter) { StringBuilder builder = new StringBuilder(); for (String value : values) { diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/fs/provider/TableFilesystemMutationProvider.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/fs/provider/TableFilesystemMutationProvider.java index 8a0e79ca58a..710ff02a609 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/fs/provider/TableFilesystemMutationProvider.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/fs/provider/TableFilesystemMutationProvider.java @@ -42,7 +42,7 @@ public class TableFilesystemMutationProvider implements FilesystemMutationProvid if (path.getSegments().size() != 1) { throw invalidOperation(); } - executor.execute("CREATE DATABASE " + path.getFileName()); + executor.execute("CREATE DATABASE " + TableFilesystemSql.identifier(path.getFileName())); } @Override @@ -61,7 +61,11 @@ public class TableFilesystemMutationProvider implements FilesystemMutationProvid if (!parent(source).equals(parent(target))) { throw invalidOperation(); } - executor.execute("ALTER TABLE " + toTablePath(source) + " RENAME TO " + tableName(target)); + executor.execute( + "ALTER TABLE " + + toTablePath(source) + + " RENAME TO " + + TableFilesystemSql.identifier(tableName(target))); } @Override @@ -72,10 +76,12 @@ public class TableFilesystemMutationProvider implements FilesystemMutationProvid if (lines == null || lines.isEmpty()) { return; } - String tablePath = toTablePath(path); List<String> statements = TableCsvAppendPlanner.plan( - tablePath, executor.query("DESC " + tablePath + " DETAILS"), lines); + databaseName(path), + tableName(path), + executor.query("DESC " + toTablePath(path) + " DETAILS"), + lines); for (String statement : statements) { executor.execute(statement); } @@ -86,8 +92,11 @@ public class TableFilesystemMutationProvider implements FilesystemMutationProvid } private static String toTablePath(FsPath path) { - List<String> segments = path.getSegments(); - return segments.get(0) + "." + tableName(path); + return TableFilesystemSql.tablePath(databaseName(path), tableName(path)); + } + + private static String databaseName(FsPath path) { + return path.getSegments().get(0); } private static boolean isDataFile(FsPath path) { diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/fs/provider/TableFilesystemSchemaProvider.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/fs/provider/TableFilesystemSchemaProvider.java index 879145b69d4..f5f60a6ab70 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/fs/provider/TableFilesystemSchemaProvider.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/fs/provider/TableFilesystemSchemaProvider.java @@ -71,7 +71,7 @@ public class TableFilesystemSchemaProvider implements FilesystemSchemaProvider { if (parseTableFile(path).kind != TableFileKind.UNKNOWN) { return new ArrayList<>(); } - return listColumns(path); + return new ArrayList<>(); } return new ArrayList<>(); } @@ -90,15 +90,8 @@ public class TableFilesystemSchemaProvider implements FilesystemSchemaProvider { if (file.kind != TableFileKind.UNKNOWN) { return describeTableFile(path, file); } - return describeTable(path); - } - String columnName = path.getFileName(); - for (FsNode node : listColumns(parent(path))) { - if (columnName.equals(node.getName())) { - return node; - } } - return new FsNode(columnName, path, FsNodeType.UNKNOWN); + return unknown(path); } @Override @@ -106,16 +99,9 @@ public class TableFilesystemSchemaProvider implements FilesystemSchemaProvider { int depth = path.getSegments().size(); TableFileRef file = parseTableFile(path); if (depth == 2 && file.kind == TableFileKind.DATA_CSV) { + ensureExists(path, file); return executor.query("SELECT * FROM " + file.toTablePath() + " LIMIT " + limit); } - if (depth == 2 && file.kind == TableFileKind.UNKNOWN) { - return executor.query("SELECT * FROM " + toTablePath(path) + " LIMIT " + limit); - } - if (depth == 3) { - String tablePath = toTablePath(parent(path)); - return executor.query( - "SELECT " + path.getFileName() + " FROM " + tablePath + " LIMIT " + limit); - } throw new SQLException("Path is not readable: " + path); } @@ -123,14 +109,18 @@ public class TableFilesystemSchemaProvider implements FilesystemSchemaProvider { public List<String> readLines(FsPath path, int limit) throws SQLException { TableFileRef file = parseTableFile(path); if (file.kind == TableFileKind.DATA_CSV) { - return rowsToCsvLines( - executor.query("SELECT * FROM " + file.toTablePath() + " LIMIT " + limit)); + ensureExists(path, file); + return head( + rowsToCsvLines(executor.query("SELECT * FROM " + file.toTablePath() + " LIMIT " + limit)), + limit); } if (file.kind == TableFileKind.SCHEMA) { - return rowsToCsvLines(executor.query("DESC " + file.toTablePath() + " DETAILS")); + ensureExists(path, file); + return head(rowsToCsvLines(executor.query("DESC " + file.toTablePath() + " DETAILS")), limit); } if (file.kind == TableFileKind.META) { - return metaLines(file); + ensureExists(path, file); + return head(metaLines(file), limit); } throw new SQLException("Path is not readable as text: " + path); } @@ -141,23 +131,10 @@ public class TableFilesystemSchemaProvider implements FilesystemSchemaProvider { TableFileRef file = parseTableFile(path); List<SqlRow> rows; if (depth == 2 && file.kind == TableFileKind.DATA_CSV) { + ensureExists(path, file); rows = executor.query( "SELECT * FROM " + file.toTablePath() + " ORDER BY time DESC LIMIT " + limit); - } else if (depth == 2 && file.kind == TableFileKind.UNKNOWN) { - rows = - executor.query( - "SELECT * FROM " + toTablePath(path) + " ORDER BY time DESC LIMIT " + limit); - } else if (depth == 3) { - String tablePath = toTablePath(parent(path)); - rows = - executor.query( - "SELECT " - + path.getFileName() - + " FROM " - + tablePath - + " ORDER BY time DESC LIMIT " - + limit); } else { throw new SQLException("Path is not readable: " + path); } @@ -168,14 +145,18 @@ public class TableFilesystemSchemaProvider implements FilesystemSchemaProvider { @Override public List<String> tailLines(FsPath path, int limit) throws SQLException { TableFileRef file = parseTableFile(path); - if (file.kind != TableFileKind.DATA_CSV) { - throw new SQLException("Path does not support tail: " + path); + if (file.kind == TableFileKind.SCHEMA || file.kind == TableFileKind.META) { + return tail(readLines(path, Integer.MAX_VALUE), limit); } - List<SqlRow> rows = - executor.query( - "SELECT * FROM " + file.toTablePath() + " ORDER BY time DESC LIMIT " + limit); - Collections.reverse(rows); - return rowsToCsvLines(rows); + if (file.kind == TableFileKind.DATA_CSV) { + ensureExists(path, file); + List<SqlRow> rows = + executor.query( + "SELECT * FROM " + file.toTablePath() + " ORDER BY time DESC LIMIT " + limit); + Collections.reverse(rows); + return tail(rowsToCsvLines(rows), limit); + } + throw new SQLException("Path does not support tail: " + path); } @Override @@ -184,16 +165,15 @@ public class TableFilesystemSchemaProvider implements FilesystemSchemaProvider { TableFileRef file = parseTableFile(path); List<SqlRow> rows; if (depth == 2 && file.kind == TableFileKind.DATA_CSV) { + ensureExists(path, file); rows = executor.query("SELECT COUNT(*) FROM " + file.toTablePath()); - } else if (depth == 2 && file.kind == TableFileKind.UNKNOWN) { - rows = executor.query("SELECT COUNT(*) FROM " + toTablePath(path)); - } else if (depth == 3) { - String tablePath = toTablePath(parent(path)); - rows = executor.query("SELECT COUNT(" + path.getFileName() + ") FROM " + tablePath); + return countValue(rows) + 1; + } else if (depth == 2 + && (file.kind == TableFileKind.SCHEMA || file.kind == TableFileKind.META)) { + return readLines(path, Integer.MAX_VALUE).size(); } else { throw new SQLException("Path is not countable: " + path); } - return countValue(rows); } @Override @@ -201,14 +181,13 @@ public class TableFilesystemSchemaProvider implements FilesystemSchemaProvider { if (paths.size() == 1) { return read(paths.get(0), limit); } - FsPath tablePath = parent(paths.get(0)); - for (FsPath path : paths) { - if (path.getSegments().size() != 3 || !tablePath.equals(parent(path))) { - throw new SQLException("Paths must be columns from the same table"); - } + throw new SQLException("Multiple paths should be read as regular files"); + } + + private void ensureExists(FsPath path, TableFileRef file) throws SQLException { + if (!tableExists(parent(path), file.table)) { + throw new SQLException("Path does not exist: " + path); } - return executor.query( - "SELECT " + columnList(paths) + " FROM " + toTablePath(tablePath) + " LIMIT " + limit); } private List<FsNode> listDatabases() throws SQLException { @@ -232,16 +211,6 @@ public class TableFilesystemSchemaProvider implements FilesystemSchemaProvider { return new FsNode(database, path, FsNodeType.UNKNOWN); } - private FsNode describeTable(FsPath path) throws SQLException { - String table = path.getFileName(); - for (FsNode node : listTables(parent(path))) { - if (table.equals(node.getName())) { - return node; - } - } - return new FsNode(table, path, FsNodeType.UNKNOWN); - } - private FsNode describeTableFile(FsPath path, TableFileRef file) throws SQLException { for (String table : listTableNames(parent(path))) { if (table.equals(file.table)) { @@ -251,18 +220,6 @@ public class TableFilesystemSchemaProvider implements FilesystemSchemaProvider { return new FsNode(path.getFileName(), path, FsNodeType.UNKNOWN); } - private List<FsNode> listTables(FsPath databasePath) throws SQLException { - List<FsNode> nodes = new ArrayList<>(); - for (String table : listTableNames(databasePath)) { - nodes.add( - new FsNode( - table, - FsPath.absolute("/" + databasePath.getFileName() + "/" + table), - FsNodeType.TABLE_TABLE)); - } - return nodes; - } - private List<FsNode> listTableFiles(FsPath databasePath) throws SQLException { String database = databasePath.getFileName(); List<FsNode> nodes = new ArrayList<>(); @@ -292,7 +249,8 @@ public class TableFilesystemSchemaProvider implements FilesystemSchemaProvider { private List<String> listTableNames(FsPath databasePath) throws SQLException { String database = databasePath.getFileName(); List<String> tables = new ArrayList<>(); - for (SqlRow row : executor.query("SHOW TABLES FROM " + database)) { + for (SqlRow row : + executor.query("SHOW TABLES FROM " + TableFilesystemSql.identifier(database))) { String table = row.get("TableName"); if (table != null) { tables.add(table); @@ -301,33 +259,20 @@ public class TableFilesystemSchemaProvider implements FilesystemSchemaProvider { return tables; } - private List<FsNode> listColumns(FsPath tablePath) throws SQLException { - List<String> segments = tablePath.getSegments(); - String database = segments.get(0); - String table = segments.get(1); - List<FsNode> nodes = new ArrayList<>(); - for (SqlRow row : executor.query("DESC " + database + "." + table + " DETAILS")) { - String column = row.get("ColumnName"); - if (column != null) { - nodes.add( - new FsNode( - column, - FsPath.absolute("/" + database + "/" + table + "/" + column), - FsNodeType.TABLE_COLUMN, - row.asMap())); + private boolean tableExists(FsPath databasePath, String table) throws SQLException { + for (String tableName : listTableNames(databasePath)) { + if (tableName.equals(table)) { + return true; } } - return nodes; - } - - private static String toTablePath(FsPath path) { - List<String> segments = path.getSegments(); - return segments.get(0) + "." + segments.get(1); + return false; } private List<String> metaLines(TableFileRef file) throws SQLException { List<SqlRow> rows = new ArrayList<>(); - for (SqlRow row : executor.query("SHOW TABLES DETAILS FROM " + file.database)) { + for (SqlRow row : + executor.query( + "SHOW TABLES DETAILS FROM " + TableFilesystemSql.identifier(file.database))) { if (file.table.equals(row.get("TableName"))) { rows.add(row); } @@ -394,17 +339,6 @@ public class TableFilesystemSchemaProvider implements FilesystemSchemaProvider { return new TableFileRef(database, table, kind).metadata(); } - private static String columnList(List<FsPath> paths) { - StringBuilder builder = new StringBuilder(); - for (FsPath path : paths) { - if (builder.length() > 0) { - builder.append(", "); - } - builder.append(path.getFileName()); - } - return builder.toString(); - } - private static long countValue(List<SqlRow> rows) { if (rows.isEmpty() || rows.get(0).asMap().isEmpty()) { return 0; @@ -424,6 +358,24 @@ public class TableFilesystemSchemaProvider implements FilesystemSchemaProvider { return FsPath.absolute(builder.toString()); } + private static FsNode unknown(FsPath path) { + return new FsNode(path.getFileName(), path, FsNodeType.UNKNOWN); + } + + private static List<String> head(List<String> lines, int limit) { + if (limit < 0 || lines.size() <= limit) { + return lines; + } + return new ArrayList<>(lines.subList(0, limit)); + } + + private static List<String> tail(List<String> lines, int limit) { + if (limit < 0 || lines.size() <= limit) { + return lines; + } + return new ArrayList<>(lines.subList(lines.size() - limit, lines.size())); + } + private static class TableFileRef { private final String database; private final String table; @@ -457,7 +409,7 @@ public class TableFilesystemSchemaProvider implements FilesystemSchemaProvider { } private String toTablePath() { - return database + "." + table; + return TableFilesystemSql.tablePath(database, table); } private FsNodeType nodeType() { diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/fs/node/FsNodeType.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/fs/provider/TableFilesystemSql.java similarity index 66% copy from iotdb-client/cli/src/main/java/org/apache/iotdb/cli/fs/node/FsNodeType.java copy to iotdb-client/cli/src/main/java/org/apache/iotdb/cli/fs/provider/TableFilesystemSql.java index df63ef5c817..9d0d549edac 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/fs/node/FsNodeType.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/fs/provider/TableFilesystemSql.java @@ -17,21 +17,20 @@ * under the License. */ -package org.apache.iotdb.cli.fs.node; +package org.apache.iotdb.cli.fs.provider; -public enum FsNodeType { - VIRTUAL_ROOT, - TREE_ROOT, - TREE_DATABASE, - TREE_INTERNAL_PATH, - TREE_DEVICE, - TREE_TIMESERIES, - TABLE_DATABASE, - TABLE_TABLE, - TABLE_VIEW, - TABLE_COLUMN, - TABLE_DATA_FILE, - TABLE_SCHEMA_FILE, - TABLE_META_FILE, - UNKNOWN +final class TableFilesystemSql { + + private TableFilesystemSql() {} + + static String tablePath(String database, String table) { + return identifier(database) + "." + identifier(table); + } + + static String identifier(String value) { + if (value.matches("[A-Za-z_][A-Za-z0-9_]*")) { + return value; + } + return "\"" + value.replace("\"", "\"\"") + "\""; + } } diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/cli/CliFilesystemModeTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/cli/CliFilesystemModeTest.java index d3d44a00972..1cbf8dcb6af 100644 --- a/iotdb-client/cli/src/test/java/org/apache/iotdb/cli/CliFilesystemModeTest.java +++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/cli/CliFilesystemModeTest.java @@ -38,6 +38,7 @@ import java.sql.SQLException; import java.sql.Statement; import static org.junit.Assert.assertFalse; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -45,8 +46,6 @@ public class CliFilesystemModeTest { @Mock private IoTDBConnection connection; @Mock private Statement statement; - @Mock private ResultSet resultSet; - @Mock private ResultSetMetaData metaData; @Mock private FilesystemShell shell; @Mock private LineReader lineReader; @@ -63,11 +62,13 @@ public class CliFilesystemModeTest { @Test public void createFilesystemShellUsesTableProviderForTableDialect() throws Exception { when(connection.getSqlDialect()).thenReturn("table"); + mockSingleColumnQuery("SHOW DATABASES", "Database", "db1"); mockSingleColumnQuery("SHOW TABLES FROM db1", "TableName", "table1"); FilesystemShell shell = Cli.createFilesystemShell(ctx, connection); shell.execute("ls /db1"); + verify(statement).executeQuery("SHOW DATABASES"); verify(statement).executeQuery("SHOW TABLES FROM db1"); } @@ -100,11 +101,13 @@ public class CliFilesystemModeTest { @Test public void createFilesystemShellUsesTreeProviderForTreeDialect() throws Exception { when(connection.getSqlDialect()).thenReturn("tree"); + mockSingleColumnQuery("SHOW DATABASES", "Database", "root.sg"); mockSingleColumnQuery("SHOW CHILD PATHS root.sg", "ChildPaths", "root.sg.d1"); FilesystemShell shell = Cli.createFilesystemShell(ctx, connection); shell.execute("ls /root/sg"); + verify(statement).executeQuery("SHOW DATABASES"); verify(statement).executeQuery("SHOW CHILD PATHS root.sg"); } @@ -122,6 +125,8 @@ public class CliFilesystemModeTest { } private void mockSingleColumnQuery(String sql, String column, String value) throws Exception { + ResultSet resultSet = mock(ResultSet.class); + ResultSetMetaData metaData = mock(ResultSetMetaData.class); when(connection.createStatement()).thenReturn(statement); when(statement.executeQuery(sql)).thenReturn(resultSet); when(resultSet.getMetaData()).thenReturn(metaData); diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/cli/fs/FilesystemShellTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/cli/fs/FilesystemShellTest.java index 5566022a31c..cc442be8be3 100644 --- a/iotdb-client/cli/src/test/java/org/apache/iotdb/cli/fs/FilesystemShellTest.java +++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/cli/fs/FilesystemShellTest.java @@ -86,6 +86,8 @@ public class FilesystemShellTest { @Test public void executeLsPrintsChildNodes() throws SQLException { + when(provider.describe(FsPath.absolute("/"))) + .thenReturn(new FsNode("/", FsPath.absolute("/"), FsNodeType.VIRTUAL_ROOT)); when(provider.list(FsPath.absolute("/"))) .thenReturn( Arrays.asList( @@ -97,27 +99,34 @@ public class FilesystemShellTest { assertEquals("root" + System.lineSeparator() + "test" + System.lineSeparator(), out.toString()); assertFalse(out.toString().contains(",")); assertFalse(out.toString().contains("TREE_ROOT")); + verify(provider).describe(FsPath.absolute("/")); verify(provider).list(FsPath.absolute("/")); } @Test public void executeLlPrintsLongListing() throws SQLException { + when(provider.describe(FsPath.absolute("/"))) + .thenReturn(new FsNode("/", FsPath.absolute("/"), FsNodeType.VIRTUAL_ROOT)); when(provider.list(FsPath.absolute("/"))) .thenReturn( Arrays.asList( new FsNode("testtest", FsPath.absolute("/testtest"), FsNodeType.TABLE_DATABASE), - new FsNode("value", FsPath.absolute("/value"), FsNodeType.TABLE_COLUMN))); + new FsNode( + "value.csv", FsPath.absolute("/value.csv"), FsNodeType.TABLE_DATA_FILE))); assertTrue(shell.execute("ll /")); assertTrue(out.toString().contains("dr-xr-xr-x")); assertTrue(out.toString().contains("-r--r--r--")); assertTrue(out.toString().contains("testtest")); + verify(provider).describe(FsPath.absolute("/")); verify(provider).list(FsPath.absolute("/")); } @Test public void executeLsLongOptionPrintsLongListing() throws SQLException { + when(provider.describe(FsPath.absolute("/"))) + .thenReturn(new FsNode("/", FsPath.absolute("/"), FsNodeType.VIRTUAL_ROOT)); when(provider.list(FsPath.absolute("/"))) .thenReturn( Arrays.asList( @@ -127,11 +136,14 @@ public class FilesystemShellTest { assertTrue(out.toString().contains("dr-xr-xr-x")); assertTrue(out.toString().contains("testtest")); + verify(provider).describe(FsPath.absolute("/")); verify(provider).list(FsPath.absolute("/")); } @Test public void executeLsAllPrintsDotEntries() throws SQLException { + when(provider.describe(FsPath.absolute("/"))) + .thenReturn(new FsNode("/", FsPath.absolute("/"), FsNodeType.VIRTUAL_ROOT)); when(provider.list(FsPath.absolute("/"))) .thenReturn( Arrays.asList( @@ -147,11 +159,14 @@ public class FilesystemShellTest { + "testtest" + System.lineSeparator(), out.toString()); + verify(provider).describe(FsPath.absolute("/")); verify(provider).list(FsPath.absolute("/")); } @Test public void executeLlAllPrintsDotEntriesInLongListing() throws SQLException { + when(provider.describe(FsPath.absolute("/"))) + .thenReturn(new FsNode("/", FsPath.absolute("/"), FsNodeType.VIRTUAL_ROOT)); when(provider.list(FsPath.absolute("/"))) .thenReturn( Arrays.asList( @@ -163,9 +178,37 @@ public class FilesystemShellTest { assertTrue(out.toString().contains("dr-xr-xr-x 1 iotdb iotdb 0 ..")); assertTrue(out.toString().contains("dr-xr-xr-x 1 iotdb iotdb 0 testtest")); assertFalse(out.toString().contains("-a")); + verify(provider).describe(FsPath.absolute("/")); verify(provider).list(FsPath.absolute("/")); } + @Test + public void executeLsRegularFilePrintsFileName() throws SQLException { + when(provider.describe(FsPath.absolute("/db1/table1.csv"))) + .thenReturn( + new FsNode( + "table1.csv", FsPath.absolute("/db1/table1.csv"), FsNodeType.TABLE_DATA_FILE)); + + assertTrue(shell.execute("ls /db1/table1.csv")); + + assertEquals("table1.csv" + System.lineSeparator(), out.toString()); + verify(provider).describe(FsPath.absolute("/db1/table1.csv")); + verify(provider, times(0)).list(FsPath.absolute("/db1/table1.csv")); + } + + @Test + public void executeLsUnknownPathPrintsNoSuchFile() throws SQLException { + when(provider.describe(FsPath.absolute("/db1/table1"))) + .thenReturn(new FsNode("table1", FsPath.absolute("/db1/table1"), FsNodeType.UNKNOWN)); + + assertTrue(shell.execute("ls /db1/table1")); + + assertEquals( + "ls: /db1/table1: No such file or directory" + System.lineSeparator(), out.toString()); + verify(provider).describe(FsPath.absolute("/db1/table1")); + verify(provider, times(0)).list(FsPath.absolute("/db1/table1")); + } + @Test public void executeCdUpdatesCurrentPath() throws SQLException { when(provider.describe(FsPath.absolute("/root"))) @@ -258,6 +301,8 @@ public class FilesystemShellTest { @Test public void executeTreePrintsChildrenUntilDepth() throws SQLException { + when(provider.describe(FsPath.absolute("/"))) + .thenReturn(new FsNode("/", FsPath.absolute("/"), FsNodeType.VIRTUAL_ROOT)); when(provider.list(FsPath.absolute("/"))) .thenReturn( Arrays.asList(new FsNode("root", FsPath.absolute("/root"), FsNodeType.TREE_ROOT))); @@ -271,20 +316,34 @@ public class FilesystemShellTest { assertTrue(out.toString().contains("sg")); assertFalse(out.toString().contains("TREE_ROOT")); assertFalse(out.toString().contains("TREE_DATABASE")); + verify(provider).describe(FsPath.absolute("/")); verify(provider).list(FsPath.absolute("/")); verify(provider).list(FsPath.absolute("/root")); } @Test - public void executeCatReadsTablePath() throws SQLException { - when(provider.read(FsPath.absolute("/db1/table1"), 20)) - .thenReturn(Arrays.asList(SqlRow.of("Time", "1", "tag1", "a", "s1", "42"))); + public void executeTreeUnknownPathPrintsNoSuchFile() throws SQLException { + when(provider.describe(FsPath.absolute("/db1/table1"))) + .thenReturn(new FsNode("table1", FsPath.absolute("/db1/table1"), FsNodeType.UNKNOWN)); + + assertTrue(shell.execute("tree /db1/table1")); + + assertEquals( + "tree: /db1/table1: No such file or directory" + System.lineSeparator(), out.toString()); + verify(provider).describe(FsPath.absolute("/db1/table1")); + verify(provider, times(0)).list(FsPath.absolute("/db1/table1")); + } + + @Test + public void executeCatReadsSchemaFileLines() throws SQLException { + when(provider.readLines(FsPath.absolute("/db1/table1.schema"), 20)) + .thenReturn(Arrays.asList("ColumnName,DataType", "key,STRING")); - assertTrue(shell.execute("cat /db1/table1")); + assertTrue(shell.execute("cat /db1/table1.schema")); - assertTrue(out.toString().contains("1\ta\t42")); - assertFalse(out.toString().contains("{")); - verify(provider).read(FsPath.absolute("/db1/table1"), 20); + assertTrue(out.toString().contains("ColumnName,DataType")); + assertTrue(out.toString().contains("key,STRING")); + verify(provider).readLines(FsPath.absolute("/db1/table1.schema"), 20); } @Test @@ -300,29 +359,37 @@ public class FilesystemShellTest { } @Test - public void executeCatReadsMultiplePathsSequentially() throws SQLException { - when(provider.read(FsPath.absolute("/db1/table1/tag1"), 20)) - .thenReturn(Arrays.asList(SqlRow.of("Time", "1", "tag1", "a"))); - when(provider.read(FsPath.absolute("/db1/table1/s1"), 20)) - .thenReturn(Arrays.asList(SqlRow.of("Time", "1", "s1", "42"))); + public void executeCatReadsMultipleTextFilesSequentially() throws SQLException { + when(provider.readLines(FsPath.absolute("/db1/table1.csv"), 20)) + .thenReturn(Arrays.asList("time,key", "1,a")); + when(provider.readLines(FsPath.absolute("/db1/table1.meta"), 20)) + .thenReturn(Arrays.asList("TableName,Status", "table1,USING")); - assertTrue(shell.execute("cat /db1/table1/tag1 /db1/table1/s1")); + assertTrue(shell.execute("cat /db1/table1.csv /db1/table1.meta")); - assertTrue(out.toString().contains("1\ta")); - assertTrue(out.toString().contains("1\t42")); - verify(provider).read(FsPath.absolute("/db1/table1/tag1"), 20); - verify(provider).read(FsPath.absolute("/db1/table1/s1"), 20); + assertEquals( + "time,key" + + System.lineSeparator() + + "1,a" + + System.lineSeparator() + + "TableName,Status" + + System.lineSeparator() + + "table1,USING" + + System.lineSeparator(), + out.toString()); + verify(provider).readLines(FsPath.absolute("/db1/table1.csv"), 20); + verify(provider).readLines(FsPath.absolute("/db1/table1.meta"), 20); } @Test - public void executeHeadReadsPathWithLimit() throws SQLException { - when(provider.read(FsPath.absolute("/db1/table1"), 5)) - .thenReturn(Arrays.asList(SqlRow.of("Time", "1", "tag1", "a", "s1", "42"))); + public void executeHeadReadsSchemaFileLinesWithLimit() throws SQLException { + when(provider.readLines(FsPath.absolute("/db1/table1.schema"), 5)) + .thenReturn(Arrays.asList("ColumnName,DataType", "key,STRING")); - assertTrue(shell.execute("head -n 5 /db1/table1")); + assertTrue(shell.execute("head -n 5 /db1/table1.schema")); - assertTrue(out.toString().contains("1\ta\t42")); - verify(provider).read(FsPath.absolute("/db1/table1"), 5); + assertTrue(out.toString().contains("ColumnName,DataType")); + verify(provider).readLines(FsPath.absolute("/db1/table1.schema"), 5); } @Test @@ -338,14 +405,14 @@ public class FilesystemShellTest { } @Test - public void executeTailReadsPathWithLimit() throws SQLException { - when(provider.tail(FsPath.absolute("/db1/table1"), 3)) - .thenReturn(Arrays.asList(SqlRow.of("Time", "2", "tag1", "b", "s1", "43"))); + public void executeTailReadsSchemaFileLinesWithLimit() throws SQLException { + when(provider.tailLines(FsPath.absolute("/db1/table1.schema"), 3)) + .thenReturn(Arrays.asList("key,STRING", "value,DOUBLE")); - assertTrue(shell.execute("tail -n 3 /db1/table1")); + assertTrue(shell.execute("tail -n 3 /db1/table1.schema")); - assertTrue(out.toString().contains("2\tb\t43")); - verify(provider).tail(FsPath.absolute("/db1/table1"), 3); + assertTrue(out.toString().contains("value,DOUBLE")); + verify(provider).tailLines(FsPath.absolute("/db1/table1.schema"), 3); } @Test @@ -384,17 +451,17 @@ public class FilesystemShellTest { @Test public void executeGrepPrintsOnlyMatchingRows() throws SQLException { - when(provider.read(FsPath.absolute("/db1/table1"), 20)) + when(provider.read(FsPath.absolute("/root/sg/d1/s1"), 20)) .thenReturn( Arrays.asList( SqlRow.of("Time", "1", "tag1", "spricoder", "s1", "42"), SqlRow.of("Time", "2", "tag1", "other", "s1", "43"))); - assertTrue(shell.execute("grep spricoder /db1/table1")); + assertTrue(shell.execute("grep spricoder /root/sg/d1/s1")); assertTrue(out.toString().contains("1\tspricoder\t42")); assertFalse(out.toString().contains("2\tother\t43")); - verify(provider).read(FsPath.absolute("/db1/table1"), 20); + verify(provider).read(FsPath.absolute("/root/sg/d1/s1"), 20); } @Test @@ -403,34 +470,36 @@ public class FilesystemShellTest { .thenReturn(new FsNode("/", FsPath.absolute("/"), FsNodeType.VIRTUAL_ROOT)); when(provider.describe(FsPath.absolute("/db1"))) .thenReturn(new FsNode("db1", FsPath.absolute("/db1"), FsNodeType.TABLE_DATABASE)); - when(provider.describe(FsPath.absolute("/db1/table1"))) - .thenReturn(new FsNode("table1", FsPath.absolute("/db1/table1"), FsNodeType.TABLE_TABLE)); + when(provider.describe(FsPath.absolute("/db1/table1.csv"))) + .thenReturn( + new FsNode( + "table1.csv", FsPath.absolute("/db1/table1.csv"), FsNodeType.TABLE_DATA_FILE)); when(provider.list(FsPath.absolute("/"))) .thenReturn( Arrays.asList(new FsNode("db1", FsPath.absolute("/db1"), FsNodeType.TABLE_DATABASE))); when(provider.list(FsPath.absolute("/db1"))) .thenReturn( Arrays.asList( - new FsNode("table1", FsPath.absolute("/db1/table1"), FsNodeType.TABLE_TABLE))); - when(provider.list(FsPath.absolute("/db1/table1"))).thenReturn(new ArrayList<>()); + new FsNode( + "table1.csv", FsPath.absolute("/db1/table1.csv"), FsNodeType.TABLE_DATA_FILE))); - assertTrue(shell.execute("find / -name table1")); + assertTrue(shell.execute("find / -name table1.csv")); - assertTrue(out.toString().contains("/db1/table1")); + assertTrue(out.toString().contains("/db1/table1.csv")); verify(provider).list(FsPath.absolute("/")); verify(provider).list(FsPath.absolute("/db1")); } @Test - public void executeLessAndMoreReadPath() throws SQLException { - when(provider.read(FsPath.absolute("/db1/table1"), 20)) - .thenReturn(Arrays.asList(SqlRow.of("Time", "1", "tag1", "a", "s1", "42"))); + public void executeLessAndMoreReadMetaFileLines() throws SQLException { + when(provider.readLines(FsPath.absolute("/db1/table1.meta"), 20)) + .thenReturn(Arrays.asList("TableName,Status", "table1,USING")); - assertTrue(shell.execute("less /db1/table1")); - assertTrue(shell.execute("more /db1/table1")); + assertTrue(shell.execute("less /db1/table1.meta")); + assertTrue(shell.execute("more /db1/table1.meta")); - assertTrue(out.toString().contains("1\ta\t42")); - verify(provider, times(2)).read(FsPath.absolute("/db1/table1"), 20); + assertTrue(out.toString().contains("table1,USING")); + verify(provider, times(2)).readLines(FsPath.absolute("/db1/table1.meta"), 20); } @Test @@ -468,12 +537,11 @@ public class FilesystemShellTest { @Test public void executeFilePrintsUnixFileType() throws SQLException { when(provider.describe(FsPath.absolute("/db1/table1"))) - .thenReturn(new FsNode("table1", FsPath.absolute("/db1/table1"), FsNodeType.TABLE_TABLE)); + .thenReturn(new FsNode("table1", FsPath.absolute("/db1/table1"), FsNodeType.UNKNOWN)); assertTrue(shell.execute("file /db1/table1")); - assertTrue(out.toString().contains("/db1/table1: directory")); - assertFalse(out.toString().contains("TABLE_TABLE")); + assertTrue(out.toString().contains("/db1/table1: unknown")); verify(provider).describe(FsPath.absolute("/db1/table1")); } @@ -488,20 +556,24 @@ public class FilesystemShellTest { } @Test - public void executePasteReadsMultiplePaths() throws SQLException { - when(provider.read( - Arrays.asList(FsPath.absolute("/db1/table1/tag1"), FsPath.absolute("/db1/table1/s1")), - 20)) - .thenReturn(Arrays.asList(SqlRow.of("Time", "1", "tag1", "a", "s1", "42"))); + public void executePasteReadsMultipleTextFilesLineByLine() throws SQLException { + when(provider.readLines(FsPath.absolute("/db1/table1.csv"), 20)) + .thenReturn(Arrays.asList("time,key", "1,a", "2,b")); + when(provider.readLines(FsPath.absolute("/db1/table2.csv"), 20)) + .thenReturn(Arrays.asList("time,value", "1,42")); - assertTrue(shell.execute("paste /db1/table1/tag1 /db1/table1/s1")); + assertTrue(shell.execute("paste /db1/table1.csv /db1/table2.csv")); - assertTrue(out.toString().contains("1\ta\t42")); - assertFalse(out.toString().contains("{")); - verify(provider) - .read( - Arrays.asList(FsPath.absolute("/db1/table1/tag1"), FsPath.absolute("/db1/table1/s1")), - 20); + assertEquals( + "time,key\ttime,value" + + System.lineSeparator() + + "1,a\t1,42" + + System.lineSeparator() + + "2,b\t" + + System.lineSeparator(), + out.toString()); + verify(provider).readLines(FsPath.absolute("/db1/table1.csv"), 20); + verify(provider).readLines(FsPath.absolute("/db1/table2.csv"), 20); } @Test @@ -541,7 +613,8 @@ public class FilesystemShellTest { .thenReturn( Arrays.asList( new FsNode("testtest", FsPath.absolute("/testtest"), FsNodeType.TABLE_DATABASE), - new FsNode("value", FsPath.absolute("/value"), FsNodeType.TABLE_COLUMN))); + new FsNode( + "value.csv", FsPath.absolute("/value.csv"), FsNodeType.TABLE_DATA_FILE))); List<String> values = complete(shell.createCompleter(), "cd t"); diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/cli/fs/command/FilesystemCommandParserTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/cli/fs/command/FilesystemCommandParserTest.java index eb97455c369..31971bf6a1e 100644 --- a/iotdb-client/cli/src/test/java/org/apache/iotdb/cli/fs/command/FilesystemCommandParserTest.java +++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/cli/fs/command/FilesystemCommandParserTest.java @@ -79,90 +79,93 @@ public class FilesystemCommandParserTest { } @Test - public void parseCatTablePath() { - FilesystemCommand command = FilesystemCommandParser.parse("cat /db1/table1"); + public void parseCatSidecarPath() { + FilesystemCommand command = FilesystemCommandParser.parse("cat /db1/table1.csv"); assertEquals(FilesystemCommand.Type.CAT, command.getType()); - assertEquals("/db1/table1", command.getPath()); + assertEquals("/db1/table1.csv", command.getPath()); } @Test public void parseCatMultiplePaths() { FilesystemCommand command = - FilesystemCommandParser.parse("cat /db1/table1/tag1 /db1/table1/s1"); + FilesystemCommandParser.parse("cat /db1/table1.csv /db1/table1.meta"); assertEquals(FilesystemCommand.Type.CAT, command.getType()); assertEquals(2, command.getPaths().size()); - assertEquals("/db1/table1/tag1", command.getPaths().get(0)); - assertEquals("/db1/table1/s1", command.getPaths().get(1)); + assertEquals("/db1/table1.csv", command.getPaths().get(0)); + assertEquals("/db1/table1.meta", command.getPaths().get(1)); } @Test public void parseHeadLimitAndPath() { - FilesystemCommand command = FilesystemCommandParser.parse("head -n 5 /db1/table1"); + FilesystemCommand command = FilesystemCommandParser.parse("head -n 5 /db1/table1.csv"); assertEquals(FilesystemCommand.Type.HEAD, command.getType()); - assertEquals("/db1/table1", command.getPath()); + assertEquals("/db1/table1.csv", command.getPath()); assertEquals(5, command.getLimit()); } @Test public void parseTailLimitAndPath() { - FilesystemCommand command = FilesystemCommandParser.parse("tail -n 3 /db1/table1"); + FilesystemCommand command = FilesystemCommandParser.parse("tail -n 3 /db1/table1.csv"); assertEquals(FilesystemCommand.Type.TAIL, command.getType()); - assertEquals("/db1/table1", command.getPath()); + assertEquals("/db1/table1.csv", command.getPath()); assertEquals(3, command.getLimit()); } @Test public void parseWcLineCountAndPath() { - FilesystemCommand command = FilesystemCommandParser.parse("wc -l /db1/table1"); + FilesystemCommand command = FilesystemCommandParser.parse("wc -l /db1/table1.csv"); assertEquals(FilesystemCommand.Type.WC, command.getType()); - assertEquals("/db1/table1", command.getPath()); + assertEquals("/db1/table1.csv", command.getPath()); assertEquals("-l", command.getOption()); } @Test public void parseGrepPatternAndPath() { - FilesystemCommand command = FilesystemCommandParser.parse("grep spricoder /db1/table1"); + FilesystemCommand command = FilesystemCommandParser.parse("grep spricoder /db1/table1.csv"); assertEquals(FilesystemCommand.Type.GREP, command.getType()); - assertEquals("/db1/table1", command.getPath()); + assertEquals("/db1/table1.csv", command.getPath()); assertEquals("spricoder", command.getPattern()); } @Test public void parseFindNamePatternAndPath() { - FilesystemCommand command = FilesystemCommandParser.parse("find /db1 -name table1"); + FilesystemCommand command = FilesystemCommandParser.parse("find /db1 -name table1.csv"); assertEquals(FilesystemCommand.Type.FIND, command.getType()); assertEquals("/db1", command.getPath()); - assertEquals("table1", command.getPattern()); + assertEquals("table1.csv", command.getPattern()); } @Test public void parseLessMoreFileAndDu() { assertEquals( - FilesystemCommand.Type.LESS, FilesystemCommandParser.parse("less /db1/table1").getType()); + FilesystemCommand.Type.LESS, + FilesystemCommandParser.parse("less /db1/table1.csv").getType()); assertEquals( - FilesystemCommand.Type.MORE, FilesystemCommandParser.parse("more /db1/table1").getType()); + FilesystemCommand.Type.MORE, + FilesystemCommandParser.parse("more /db1/table1.csv").getType()); assertEquals( - FilesystemCommand.Type.FILE, FilesystemCommandParser.parse("file /db1/table1").getType()); + FilesystemCommand.Type.FILE, + FilesystemCommandParser.parse("file /db1/table1.csv").getType()); assertEquals( - FilesystemCommand.Type.DU, FilesystemCommandParser.parse("du /db1/table1").getType()); + FilesystemCommand.Type.DU, FilesystemCommandParser.parse("du /db1/table1.csv").getType()); } @Test public void parsePastePaths() { FilesystemCommand command = - FilesystemCommandParser.parse("paste /db1/table1/tag1 /db1/table1/s1"); + FilesystemCommandParser.parse("paste /db1/table1.csv /db1/table2.csv"); assertEquals(FilesystemCommand.Type.PASTE, command.getType()); assertEquals(2, command.getPaths().size()); - assertEquals("/db1/table1/tag1", command.getPaths().get(0)); - assertEquals("/db1/table1/s1", command.getPaths().get(1)); + assertEquals("/db1/table1.csv", command.getPaths().get(0)); + assertEquals("/db1/table2.csv", command.getPaths().get(1)); } @Test @@ -220,15 +223,15 @@ public class FilesystemCommandParserTest { assertEquals(FilesystemCommand.Type.MKDIR, mkdir.getType()); assertEquals("/db1", mkdir.getPath()); - FilesystemCommand rm = FilesystemCommandParser.parse("rm /db1/table1"); + FilesystemCommand rm = FilesystemCommandParser.parse("rm /db1/table1.csv"); assertEquals(FilesystemCommand.Type.RM, rm.getType()); - assertEquals("/db1/table1", rm.getPath()); + assertEquals("/db1/table1.csv", rm.getPath()); - FilesystemCommand mv = FilesystemCommandParser.parse("mv /db1/table1 /db1/table2"); + FilesystemCommand mv = FilesystemCommandParser.parse("mv /db1/table1.csv /db1/table2.csv"); assertEquals(FilesystemCommand.Type.MV, mv.getType()); assertEquals(2, mv.getPaths().size()); - assertEquals("/db1/table1", mv.getPaths().get(0)); - assertEquals("/db1/table2", mv.getPaths().get(1)); + assertEquals("/db1/table1.csv", mv.getPaths().get(0)); + assertEquals("/db1/table2.csv", mv.getPaths().get(1)); } @Test diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/cli/fs/provider/TableFilesystemMutationProviderTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/cli/fs/provider/TableFilesystemMutationProviderTest.java index 2f23c1d4f1d..17efc72a76a 100644 --- a/iotdb-client/cli/src/test/java/org/apache/iotdb/cli/fs/provider/TableFilesystemMutationProviderTest.java +++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/cli/fs/provider/TableFilesystemMutationProviderTest.java @@ -68,7 +68,7 @@ public class TableFilesystemMutationProviderTest { } @Test - public void removeRejectsRootDatabaseSchemaMetaAndLegacyTableLevel() throws SQLException { + public void removeRejectsRootDatabaseSchemaMetaBareTableAndColumnPaths() throws SQLException { assertInvalidOperation(() -> provider.remove(FsPath.absolute("/"))); assertInvalidOperation(() -> provider.remove(FsPath.absolute("/db1"))); assertInvalidOperation(() -> provider.remove(FsPath.absolute("/db1/table1"))); @@ -134,6 +134,17 @@ public class TableFilesystemMutationProviderTest { verify(executor).execute("INSERT INTO db1.table1(time, value) VALUES (1, 2.0)"); } + @Test + public void appendCsvQuotesSpecialIdentifiers() throws SQLException { + mockSpecialTableSchema(); + + provider.append(FsPath.absolute("/db-1/table-1.csv"), Arrays.asList("time,key-value", "1,a")); + + verify(executor).query("DESC \"db-1\".\"table-1\" DETAILS"); + verify(executor) + .execute("INSERT INTO \"db-1\".\"table-1\"(time, \"key-value\") VALUES (1, 'a')"); + } + @Test public void appendRejectsSidecarAndMissingTime() throws SQLException { assertInvalidOperation( @@ -155,6 +166,16 @@ public class TableFilesystemMutationProviderTest { "ColumnName", "value", "DataType", "DOUBLE"))); } + private void mockSpecialTableSchema() throws SQLException { + when(executor.query("DESC \"db-1\".\"table-1\" DETAILS")) + .thenReturn( + org.apache.iotdb.cli.fs.sql.SqlRow.list( + org.apache.iotdb.cli.fs.sql.SqlRow.of( + "ColumnName", "time", "DataType", "TIMESTAMP"), + org.apache.iotdb.cli.fs.sql.SqlRow.of( + "ColumnName", "key-value", "DataType", "STRING"))); + } + private static void assertInvalidOperation(SqlOperation operation) throws SQLException { try { operation.run(); diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/cli/fs/provider/TableFilesystemSchemaProviderTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/cli/fs/provider/TableFilesystemSchemaProviderTest.java index b99a30761d7..cbf254b0eef 100644 --- a/iotdb-client/cli/src/test/java/org/apache/iotdb/cli/fs/provider/TableFilesystemSchemaProviderTest.java +++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/cli/fs/provider/TableFilesystemSchemaProviderTest.java @@ -31,11 +31,13 @@ import org.mockito.Mock; import org.mockito.MockitoAnnotations; import java.sql.SQLException; -import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; public class TableFilesystemSchemaProviderTest { @@ -97,37 +99,24 @@ public class TableFilesystemSchemaProviderTest { } @Test - public void listTableReturnsColumns() throws SQLException { - when(executor.query("DESC db1.table1 DETAILS")) - .thenReturn( - SqlRow.list( - SqlRow.of("ColumnName", "tag1", "DataType", "STRING", "Category", "TAG"), - SqlRow.of("ColumnName", "s1", "DataType", "INT32", "Category", "FIELD"))); - + public void bareTablePathIsNotDirectoryInSidecarModel() throws SQLException { List<FsNode> children = provider.list(FsPath.absolute("/db1/table1")); + FsNode node = provider.describe(FsPath.absolute("/db1/table1")); - assertEquals(2, children.size()); - assertEquals("tag1", children.get(0).getName()); - assertEquals(FsNodeType.TABLE_COLUMN, children.get(0).getType()); - assertEquals("TAG", children.get(0).getMetadata().get("Category")); - verify(executor).query("DESC db1.table1 DETAILS"); + assertEquals(0, children.size()); + assertEquals("table1", node.getName()); + assertEquals(FsNodeType.UNKNOWN, node.getType()); + verifyZeroInteractions(executor); } @Test - public void describeColumnReturnsColumnMetadata() throws SQLException { - when(executor.query("DESC db1.table1 DETAILS")) - .thenReturn( - SqlRow.list( - SqlRow.of("ColumnName", "tag1", "DataType", "STRING", "Category", "TAG"), - SqlRow.of("ColumnName", "s1", "DataType", "INT32", "Category", "FIELD"))); - + public void columnPathIsNotAFileInSidecarModel() throws SQLException { FsNode node = provider.describe(FsPath.absolute("/db1/table1/s1")); assertEquals("s1", node.getName()); assertEquals("/db1/table1/s1", node.getPath().toString()); - assertEquals(FsNodeType.TABLE_COLUMN, node.getType()); - assertEquals("INT32", node.getMetadata().get("DataType")); - verify(executor).query("DESC db1.table1 DETAILS"); + assertEquals(FsNodeType.UNKNOWN, node.getType()); + verifyZeroInteractions(executor); } @Test @@ -152,45 +141,18 @@ public class TableFilesystemSchemaProviderTest { } @Test - public void describeTableReturnsDirectoryNode() throws SQLException { - when(executor.query("SHOW TABLES FROM db1")) - .thenReturn(SqlRow.list(SqlRow.of("TableName", "table1"))); - - FsNode node = provider.describe(FsPath.absolute("/db1/table1")); - - assertEquals("table1", node.getName()); - assertEquals("/db1/table1", node.getPath().toString()); - assertEquals(FsNodeType.TABLE_TABLE, node.getType()); - verify(executor).query("SHOW TABLES FROM db1"); - } - - @Test - public void readColumnSelectsColumnFromTable() throws SQLException { - when(executor.query("SELECT s1 FROM db1.table1 LIMIT 5")) - .thenReturn(SqlRow.list(SqlRow.of("Time", "1", "s1", "42"))); - - List<SqlRow> rows = provider.read(FsPath.absolute("/db1/table1/s1"), 5); - - assertEquals(1, rows.size()); - assertEquals("42", rows.get(0).get("s1")); - verify(executor).query("SELECT s1 FROM db1.table1 LIMIT 5"); - } - - @Test - public void readTableSelectsAllColumnsFromTable() throws SQLException { - when(executor.query("SELECT * FROM db1.table1 LIMIT 5")) - .thenReturn(SqlRow.list(SqlRow.of("Time", "1", "tag1", "a", "s1", "42"))); - - List<SqlRow> rows = provider.read(FsPath.absolute("/db1/table1"), 5); - - assertEquals(1, rows.size()); - assertEquals("a", rows.get(0).get("tag1")); - assertEquals("42", rows.get(0).get("s1")); - verify(executor).query("SELECT * FROM db1.table1 LIMIT 5"); + public void readBareTableAndColumnPathsAreRejected() throws SQLException { + assertSqlError( + () -> provider.read(FsPath.absolute("/db1/table1"), 5), + "Path is not readable: /db1/table1"); + assertSqlError( + () -> provider.read(FsPath.absolute("/db1/table1/s1"), 5), + "Path is not readable: /db1/table1/s1"); } @Test public void readTableCsvReturnsCsvLinesWithHeader() throws SQLException { + mockTableExists("db1", "table1"); when(executor.query("SELECT * FROM db1.table1 LIMIT 5")) .thenReturn(SqlRow.list(SqlRow.of("Time", "1", "tag1", "a", "s1", "42"))); @@ -201,8 +163,24 @@ public class TableFilesystemSchemaProviderTest { verify(executor).query("SELECT * FROM db1.table1 LIMIT 5"); } + @Test + public void readTableCsvQuotesSpecialIdentifiers() throws SQLException { + when(executor.query("SHOW TABLES FROM \"db-1\"")) + .thenReturn(SqlRow.list(SqlRow.of("TableName", "table-1"))); + when(executor.query("SELECT * FROM \"db-1\".\"table-1\" LIMIT 5")) + .thenReturn(SqlRow.list(SqlRow.of("Time", "1", "tag1", "a"))); + + List<String> lines = provider.readLines(FsPath.absolute("/db-1/table-1.csv"), 5); + + assertEquals("Time,tag1", lines.get(0)); + assertEquals("1,a", lines.get(1)); + verify(executor).query("SHOW TABLES FROM \"db-1\""); + verify(executor).query("SELECT * FROM \"db-1\".\"table-1\" LIMIT 5"); + } + @Test public void readTableSchemaReturnsIoTDBDescCsvLines() throws SQLException { + mockTableExists("db1", "table1"); when(executor.query("DESC db1.table1 DETAILS")) .thenReturn( SqlRow.list( @@ -219,6 +197,7 @@ public class TableFilesystemSchemaProviderTest { @Test public void readTableMetaReturnsIoTDBTableCsvLines() throws SQLException { + mockTableExists("db1", "table1"); when(executor.query("SHOW TABLES DETAILS FROM db1")) .thenReturn( SqlRow.list( @@ -250,44 +229,101 @@ public class TableFilesystemSchemaProviderTest { } @Test - public void tailTableSelectsNewestRowsAndReturnsOriginalOrder() throws SQLException { + public void tailTableCsvReturnsLastPhysicalLines() throws SQLException { + mockTableExists("db1", "table1"); when(executor.query("SELECT * FROM db1.table1 ORDER BY time DESC LIMIT 2")) .thenReturn( SqlRow.list( - SqlRow.of("Time", "2", "tag1", "b", "s1", "43"), - SqlRow.of("Time", "1", "tag1", "a", "s1", "42"))); + SqlRow.of("Time", "3", "tag1", "c", "s1", "44"), + SqlRow.of("Time", "2", "tag1", "b", "s1", "43"))); - List<SqlRow> rows = provider.tail(FsPath.absolute("/db1/table1"), 2); + List<String> lines = provider.tailLines(FsPath.absolute("/db1/table1.csv"), 2); - assertEquals("1", rows.get(0).get("Time")); - assertEquals("2", rows.get(1).get("Time")); + assertEquals("2,b,43", lines.get(0)); + assertEquals("3,c,44", lines.get(1)); verify(executor).query("SELECT * FROM db1.table1 ORDER BY time DESC LIMIT 2"); } @Test - public void countTableSelectsRowCount() throws SQLException { + public void countTableCsvCountsHeaderAndRows() throws SQLException { + mockTableExists("db1", "table1"); when(executor.query("SELECT COUNT(*) FROM db1.table1")) .thenReturn(SqlRow.list(SqlRow.of("count", "2"))); long count = provider.count(FsPath.absolute("/db1/table1.csv")); - assertEquals(2L, count); + assertEquals(3L, count); verify(executor).query("SELECT COUNT(*) FROM db1.table1"); } @Test - public void readColumnsSelectsMultipleColumnsFromSameTable() throws SQLException { - when(executor.query("SELECT tag1, s1 FROM db1.table1 LIMIT 5")) - .thenReturn(SqlRow.list(SqlRow.of("Time", "1", "tag1", "a", "s1", "42"))); + public void countSchemaAndMetaCountsCsvLines() throws SQLException { + mockTableExists("db1", "table1"); + when(executor.query("DESC db1.table1 DETAILS")) + .thenReturn( + SqlRow.list( + SqlRow.of("ColumnName", "tag1", "DataType", "STRING", "Category", "TAG"), + SqlRow.of("ColumnName", "s1", "DataType", "INT32", "Category", "FIELD"))); + when(executor.query("SHOW TABLES DETAILS FROM db1")) + .thenReturn( + SqlRow.list( + SqlRow.of("TableName", "table1", "Status", "USING"), + SqlRow.of("TableName", "table2", "Status", "USING"))); - List<SqlRow> rows = - provider.read( - Arrays.asList(FsPath.absolute("/db1/table1/tag1"), FsPath.absolute("/db1/table1/s1")), - 5); + assertEquals(3L, provider.count(FsPath.absolute("/db1/table1.schema"))); + assertEquals(2L, provider.count(FsPath.absolute("/db1/table1.meta"))); + verify(executor).query("DESC db1.table1 DETAILS"); + verify(executor).query("SHOW TABLES DETAILS FROM db1"); + } + + @Test + public void missingSidecarFilesAreRejectedBeforeReadingContent() throws SQLException { + when(executor.query("SHOW TABLES FROM db1")) + .thenReturn(SqlRow.list(SqlRow.of("TableName", "table2"))); + + assertSqlError( + () -> provider.readLines(FsPath.absolute("/db1/table1.csv"), 5), + "Path does not exist: /db1/table1.csv"); + assertSqlError( + () -> provider.readLines(FsPath.absolute("/db1/table1.schema"), 5), + "Path does not exist: /db1/table1.schema"); + assertSqlError( + () -> provider.readLines(FsPath.absolute("/db1/table1.meta"), 5), + "Path does not exist: /db1/table1.meta"); + + verify(executor, times(3)).query("SHOW TABLES FROM db1"); + } + + @Test + public void missingTableCsvTailAndCountAreRejectedBeforeDataQueries() throws SQLException { + when(executor.query("SHOW TABLES FROM db1")) + .thenReturn(SqlRow.list(SqlRow.of("TableName", "table2"))); + + assertSqlError( + () -> provider.tailLines(FsPath.absolute("/db1/table1.csv"), 2), + "Path does not exist: /db1/table1.csv"); + assertSqlError( + () -> provider.count(FsPath.absolute("/db1/table1.csv")), + "Path does not exist: /db1/table1.csv"); + + verify(executor, times(2)).query("SHOW TABLES FROM db1"); + } + + private static void assertSqlError(SqlOperation operation, String message) throws SQLException { + try { + operation.run(); + fail(); + } catch (SQLException e) { + assertEquals(message, e.getMessage()); + } + } + + private void mockTableExists(String database, String table) throws SQLException { + when(executor.query("SHOW TABLES FROM " + database)) + .thenReturn(SqlRow.list(SqlRow.of("TableName", table))); + } - assertEquals(1, rows.size()); - assertEquals("a", rows.get(0).get("tag1")); - assertEquals("42", rows.get(0).get("s1")); - verify(executor).query("SELECT tag1, s1 FROM db1.table1 LIMIT 5"); + private interface SqlOperation { + void run() throws SQLException; } }
