Improve Filtering (#397)

Author: Marc-Andre-Rivet

.circleci/config.yml

@@ -1,7 +1,7 @@
 version: 2
 
 jobs:
-    "test-v0":
+    "test":
         docker:
             - image: circleci/python:3.6.7-node-browsers
             - image: cypress/base:10
@@ -30,67 +30,7 @@ jobs:
                     sudo pip install --upgrade virtualenv
                     python -m venv venv || virtualenv venv
                     . venv/bin/activate
-                    pip install -r requirements-base.txt --quiet
-
-            - run:
-                name: Install dependencies (dash)
-                command: |
-                    git clone [email protected]:plotly/dash.git
-                    git clone [email protected]:plotly/dash-renderer.git
-                    git clone [email protected]:plotly/dash-core-components.git
-                    git clone [email protected]:plotly/dash-html-components.git
-                    . venv/bin/activate
-                    pip install -e ./dash --quiet
-                    cd dash-renderer && npm install --ignore-scripts && npm run build && pip install -e . && cd ..
-                    cd dash-core-components && npm install --ignore-scripts && npm run build && pip install -e . && cd ..
-                    cd dash-html-components && npm install --ignore-scripts && npm run build && pip install -e . && cd ..
-
-            - run:
-                name: Build
-                command: |
-                    . venv/bin/activate
-                    npm run private::build:js-test
-                    npm run private::build:py
-                    pip install -e .
-
-            - run:
-                name: Run tests
-                command: |
-                    . venv/bin/activate
-                    npm run test-v0
-
-
-    "test-v1":
-        docker:
-            - image: circleci/python:3.6.7-node-browsers
-            - image: cypress/base:10
-
-        steps:
-            - checkout
-            - restore_cache:
-                key: deps1-{{ .Branch }}-{{ checksum "package-lock.json" }}-{{ checksum "package.json" }}-{{ checksum ".circleci/config.yml" }}
-            - run:
-                name: Install npm packages
-                command: npm install
-            - run:
-                name: Cypress Install
-                command: |
-                    $(npm bin)/cypress install
-
-            - save_cache:
-                key: deps1-{{ .Branch }}-{{ checksum "package-lock.json" }}-{{ checksum "package.json" }}-{{ checksum ".circleci/config.yml" }}
-                paths:
-                    - node_modules
-                    - /home/circleci/.cache/Cypress
-
-            - run:
-                name: Install requirements
-                command: |
-                    sudo pip install --upgrade virtualenv
-                    python -m venv venv || virtualenv venv
-                    . venv/bin/activate
-                    pip install -r requirements-base.txt --quiet
-                    pip install -r requirements-v1.txt --quiet
+                    pip install -r requirements.txt --quiet
 
             - run:
                 name: Build
@@ -104,7 +44,7 @@ jobs:
                 name: Run tests
                 command: |
                     . venv/bin/activate
-                    npm run test-v1
+                    npm run test
 
 
     "visual-test":
@@ -172,7 +112,7 @@ jobs:
                 when: always
 
 
-    "python-3.6-v0":
+    "python-3.6":
         docker:
             - image: circleci/python:3.6.7-stretch-node-browsers
 
@@ -224,53 +164,11 @@ jobs:
                   python -m unittest tests.dash.test_integration
 
 
-    "python-3.6-v1":
-        docker:
-            - image: circleci/python:3.6.7-stretch-node-browsers
-
-        environment:
-            PERCY_ENABLED: True
-            PERCY_PROJECT: plotly/dash-table-python-v1
-
-        steps:
-            - checkout
-
-            - run:
-                name: Inject Percy Environment variables
-                command: |
-                  echo 'export PERCY_TOKEN="$PERCY_PYTHON_TOKEN_V1"' >> $BASH_ENV
-
-            - run:
-                name: Install requirements
-                command: |
-                    sudo pip install --upgrade virtualenv
-                    python -m venv venv || virtualenv venv
-                    . venv/bin/activate
-                    pip install -r requirements-base.txt --quiet
-                    pip install -r requirements-v1.txt --quiet
-                    npm install
-
-            - run:
-                name: Install test requirements
-                command: |
-                    . venv/bin/activate
-                    npm run build
-                    pip install -e .
-
-            - run:
-                name: Run integration tests
-                command: |
-                  . venv/bin/activate
-                  python -m unittest tests.dash.test_integration
-
-
 workflows:
     version: 2
     build:
         jobs:
-            - "python-3.6-v0"
-            - "python-3.6-v1"
+            - "python-3.6"
             - "node"
-            - "test-v0"
-            - "test-v1"
+            - "test"
             - "visual-test"

.config/webpack/base.js

@@ -43,6 +43,10 @@ module.exports = (preprocessor = {}, mode = 'development') => {
                     test: /demo[/\\]index.html?$/,
                     loader: 'file-loader?name=index.[ext]'
                 },
+                {
+                    test: /\.csv$/,
+                    loader: 'raw-loader'
+                },
                 {
                     test: /\.ts(x?)$/,
                     exclude: /node_modules/,

CHANGELOG.md

@@ -2,6 +2,22 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## [Unreleased]
+### Added
+[#397](https://github.com/plotly/dash-table/pull/397)
+- Improve filtering syntax and capabilities
+    - new field syntax `{myField}`
+    - short form by-column filter
+        - implicit column and `eq` operator (e.g `"value"`)
+        - implicit column (e.g `ne "value"`)
+        - explicit form (e.g `{field} ne "value"`)
+    - new `contains` relational operator for strings
+    - new readonly `derived_filter_structure` prop exposing the query structure in a programmatically friendlier way
+
+### Changed
+[#397](https://github.com/plotly/dash-table/pull/397)
+- Rename `filtering_settings` to `filter`
+
 ## [3.6.0] - 2019-03-04
 ### Fixed
 [#189](https://github.com/plotly/dash-table/issues/189)

demo/App.js

@@ -14,6 +14,7 @@ class App extends Component {
         super();
 
         this.state = AppState;
+        this.state.temp_filtering = '';
 
         const setProps = memoizeOne(() => {
             return newProps => {
@@ -33,15 +34,30 @@ class App extends Component {
         const mode = Environment.searchParams.get('mode');
 
         if (mode === AppMode.Filtering) {
-            return (<button
-                className='clear-filters'
-                onClick={() => {
-                    const tableProps = R.clone(this.state.tableProps);
-                    tableProps.filtering_settings = '';
-
-                    this.setState({ tableProps });
-                }}
-            >Clear Filter</button>);
+            return (
+                <div>
+                    <button
+                        className='clear-filters'
+                        onClick={() => {
+                            const tableProps = R.clone(this.state.tableProps);
+                            tableProps.filter = '';
+
+                            this.setState({ tableProps });
+                        }}
+                    >Clear Filter</button>
+                    <input
+                        style={{ width: '500px' }}
+                        value={this.state.temp_filtering}
+                        onChange={
+                            e => this.setState({ temp_filtering: e.target.value })
+                        }
+                        onBlur={e => {
+                            const tableProps = R.clone(this.state.tableProps);
+                            tableProps.filter = e.target.value;
+
+                            this.setState({ tableProps });
+                        }} />
+                </div>);
         }
     }
 

demo/AppMode.ts

@@ -65,7 +65,7 @@ function getBaseTableProps(mock: IDataMock) {
             max_width: '1000px',
             width: '1000px'
         },
-        style_data_conditional: [
+        style_cell_conditional: [
             { max_width: 150, min_width: 150, width: 150 },
             { if: { column_id: 'rows' }, max_width: 60, min_width: 60, width: 60 },
             { if: { column_id: 'bbb' }, max_width: 200, min_width: 200, width: 200 },
@@ -150,7 +150,7 @@ function getTooltipsState() {
     state.tableProps.column_conditional_tooltips = [{
         if: {
             column_id: 'aaa-readonly',
-            filter: `aaa is prime`
+            filter: `{aaa} is prime`
         },
         type: TooltipSyntax.Markdown,
         value: `### Go Proverbs\nCapture three to get an eye`

package.json

@@ -30,13 +30,11 @@
     "private::runtests:unit": "cypress run --browser chrome --spec 'tests/cypress/tests/unit/**/*'",
     "private::runtests:standalone": "cypress run --browser chrome --spec 'tests/cypress/tests/standalone/**/*'",
     "private::runtests:server": "cypress run --browser chrome --spec 'tests/cypress/tests/server/**/*'",
-    "private::runtests-v0": "run-s private::runtests:server",
-    "private::runtests-v1": "run-s private::runtests:python private::runtests:unit private::runtests:standalone private::runtests:server",
+    "private::runtests": "run-s private::runtests:python private::runtests:unit private::runtests:standalone private::runtests:server",
     "build.watch": "webpack-dev-server --content-base dash_table --mode development",
     "build": "run-s private::build:js private::build:py",
     "lint": "run-s private::lint:*",
-    "test-v0": "run-p --race private::host* private::runtests-v0",
-    "test-v1": "run-p --race private::host* private::runtests-v1",
+    "test": "run-p --race private::host* private::runtests",
     "test.visual": "build-storybook && percy-storybook",
     "test.visual-local": "build-storybook",
     "test.watch": "run-p --race \"private::build:js-test-watch\" --race private::host* private::opentests"
@@ -55,6 +53,7 @@
     "@storybook/cli": "^5.0.5",
     "@storybook/react": "^5.0.5",
     "@types/d3-format": "^1.3.1",
+    "@types/papaparse": "^4.5.9",
     "@types/ramda": "^0.26.5",
     "@types/react": "^16.8.8",
     "@types/react-dom": "^16.8.3",
@@ -71,7 +70,9 @@
     "less-loader": "^4.1.0",
     "npm": "^6.9.0",
     "npm-run-all": "^4.1.5",
+    "papaparse": "^4.6.3",
     "ramda": "^0.26.1",
+    "raw-loader": "^2.0.0",
     "react": "16.8.5",
     "react-docgen": "^4.1.0",
     "react-dom": "16.8.5",

requirements-v1.txt

@@ -1,2 +1,2 @@
 -r requirements-base.txt
--r requirements-v1.txt
+-r requirements-v0.txt

requirements.txt

@@ -1,20 +1,63 @@
+import * as R from 'ramda';
+
 import Logger from 'core/Logger';
-import lexer from 'core/syntax-tree/lexer';
-import syntaxer, { ISyntaxerResult } from 'core/syntax-tree/syntaxer';
+import lexer, { ILexerResult } from 'core/syntax-tree/lexer';
+import syntaxer, { ISyntaxerResult, ISyntaxTree } from 'core/syntax-tree/syntaxer';
+import { Lexicon } from './lexicon';
+
+interface IStructure {
+    subType?: string;
+    type: string;
+    value: any;
+
+    block?: IStructure;
+    left?: IStructure;
+    right?: IStructure;
+}
+
+function toStructure(tree: ISyntaxTree): IStructure {
+    const { block, left, lexeme, right, value } = tree;
+
+    const res: IStructure = {
+        subType: lexeme.subType,
+        type: lexeme.type,
+        value: lexeme.present ? lexeme.present(tree) : value
+    };
+
+    if (block) {
+        res.block = toStructure(block);
+    }
+
+    if (left) {
+        res.left = toStructure(left);
+    }
+
+    if (right) {
+        res.right = toStructure(right);
+    }
+
+    return res;
+}
 
 export default class SyntaxTree {
-    private result: ISyntaxerResult;
+    protected lexerResult: ILexerResult;
+    protected syntaxerResult: ISyntaxerResult;
 
     get isValid() {
-        return this.result.valid;
+        return this.syntaxerResult.valid;
     }
 
     private get tree() {
-        return this.result.tree;
+        return this.syntaxerResult.tree;
     }
 
-    constructor(private readonly query: string) {
-        this.result = syntaxer(lexer(this.query));
+    constructor(
+        public readonly lexicon: Lexicon,
+        public readonly query: string,
+        postProcessor: (res: ILexerResult) => ILexerResult = res => res
+    ) {
+        this.lexerResult = postProcessor(lexer(this.lexicon, this.query));
+        this.syntaxerResult = syntaxer(this.lexerResult);
     }
 
     evaluate = (target: any) => {
@@ -33,4 +76,18 @@ export default class SyntaxTree {
     filter = (targets: any[]) => {
         return targets.filter(this.evaluate);
     }
+
+    toQueryString() {
+        return this.lexerResult.valid ?
+            R.map(l => l.value, this.lexerResult.lexemes).join(' ') :
+            '';
+    }
+
+    toStructure() {
+        if (!this.isValid || !this.syntaxerResult.tree) {
+            return null;
+        }
+
+        return toStructure(this.syntaxerResult.tree);
+    }
 }