feat: can now query historical data (#752)

* 'at' to recordsAggregation pipeline

* working at parameter

* working at parameter

---------

Co-authored-by: Antoine Hurard <[email protected]>

Author: estelafs

src/models/layer.model.ts

@@ -131,6 +131,7 @@ export interface Layer extends Document {
   layerDefinition?: LayerDefinition;
   popupInfo?: PopupElement[];
   contextFilters: string;
+  at: string;
 }
 
 /** Mongoose layer schema declaration */
@@ -185,6 +186,7 @@ const layerSchema = new Schema(
       },
     },
     contextFilters: String,
+    at: String,
   },
   {
     timestamps: { createdAt: 'createdAt', updatedAt: 'modifiedAt' },

src/routes/gis/index.ts

@@ -247,6 +247,7 @@ router.get('/feature', async (req, res) => {
   const contextFilters = JSON.parse(
     decodeURIComponent(get(req, 'query.contextFilters', null))
   );
+  const at = get(req, 'query.at') as string | undefined;
   // const tolerance = get(req, 'query.tolerance', 1);
   // const highQuality = get(req, 'query.highquality', true);
   // turf.simplify(geoJsonData, {
@@ -319,14 +320,15 @@ router.get('/feature', async (req, res) => {
       // const filterPolygon = getFilterPolygon(req.query);
 
       if (aggregation) {
-        query = `query recordsAggregation($resource: ID!, $aggregation: ID!, $contextFilters: JSON, $first: Int) {
-          recordsAggregation(resource: $resource, aggregation: $aggregation, contextFilters: $contextFilters, first: $first)
+        query = `query recordsAggregation($resource: ID!, $aggregation: ID!, $contextFilters: JSON, $first: Int, $at: Date) {
+          recordsAggregation(resource: $resource, aggregation: $aggregation, contextFilters: $contextFilters, first: $first, at: $at)
         }`;
         variables = {
           resource: resourceData._id,
           aggregation: aggregation._id,
           contextFilters,
           first: 1000,
+          at: at ? new Date(at) : undefined,
         };
       } else if (layout) {
         query = buildQuery(layout.query);
@@ -338,6 +340,7 @@ router.get('/feature', async (req, res) => {
               ? [layout.query.filter, contextFilters]
               : [layout.query.filter],
           },
+          at: at ? new Date(at) : undefined,
         };
       } else {
         return res.status(404).send(i18next.t('common.errors.dataNotFound'));

src/schema/inputs/layer.input.ts

@@ -168,6 +168,7 @@ const LayerInputType = new GraphQLInputObjectType({
     },
     datasource: { type: LayerDataSourceInputType },
     contextFilters: { type: GraphQLString },
+    at: { type: GraphQLString },
   }),
 });
 

src/schema/query/recordsAggregation.query.ts

@@ -22,6 +22,7 @@ import { logger } from '@services/logger.service';
 import buildCalculatedFieldPipeline from '../../utils/aggregation/buildCalculatedFieldPipeline';
 import checkPageSize from '@utils/schema/errors/checkPageSize.util';
 import { accessibleBy } from '@casl/mongoose';
+import { GraphQLDate } from 'graphql-scalars';
 
 /** Pagination default items per query */
 const DEFAULT_FIRST = 10;
@@ -63,6 +64,66 @@ const extractSourceFields = (filter: any, fields: string[] = []) => {
   }
 };
 
+/**
+ * Build At aggregation, filtering out items created after this date, and using version that matches date
+ *
+ * @param at Date
+ * @returns At aggregation
+ */
+const getAtAggregation = (at: Date) => {
+  return [
+    {
+      $match: {
+        createdAt: {
+          $lte: at,
+        },
+      },
+    },
+    {
+      $lookup: {
+        from: 'versions',
+        localField: 'versions',
+        foreignField: '_id',
+        pipeline: [
+          {
+            $match: {
+              createdAt: {
+                $lte: at,
+              },
+            },
+          },
+          {
+            $sort: {
+              createdAt: -1,
+            },
+          },
+          {
+            $limit: 1,
+          },
+        ],
+        as: '__version',
+      },
+    },
+    {
+      $unwind: {
+        path: '$__version',
+        preserveNullAndEmptyArrays: true,
+      },
+    },
+    {
+      $addFields: {
+        data: {
+          $cond: {
+            if: { $ifNull: ['$__version', false] },
+            then: '$__version.data',
+            else: '$data',
+          },
+        },
+      },
+    },
+  ];
+};
+
 /**
  * Take an aggregation configuration as parameter.
  * Return aggregated records data.
@@ -78,6 +139,7 @@ export default {
     skip: { type: GraphQLInt },
     sortField: { type: GraphQLString },
     sortOrder: { type: GraphQLString },
+    at: { type: GraphQLDate },
   },
   async resolve(parent, args, context) {
     // Make sure that the page size is not too important
@@ -481,6 +543,7 @@ export default {
               $and: [mongooseFilter, permissionFilters],
             },
           },
+          ...(args.at ? getAtAggregation(new Date(args.at)) : []),
         ]
       );
       // Build pipeline stages

src/schema/types/layer.type.ts

@@ -187,6 +187,7 @@ export const LayerType = new GraphQLObjectType({
     layerType: { type: LayerTypeEnum },
     datasource: { type: LayerDatasource },
     contextFilters: { type: GraphQLString },
+    at: { type: GraphQLString },
   }),
 });
 

src/utils/query/queryBuilder.ts

@@ -146,14 +146,15 @@ export const buildQuery = (query: any): any => {
   if (query && query.fields.length > 0) {
     const fields = ['canUpdate\ncanDelete\n'].concat(buildFields(query.fields));
     const gqlQuery = `
-      query GetCustomQuery($first: Int, $skip: Int, $filter: JSON, $sortField: String, $sortOrder: String, $display: Boolean) {
+      query GetCustomQuery($first: Int, $skip: Int, $filter: JSON, $sortField: String, $sortOrder: String, $display: Boolean, $at: Date) {
         ${query.name}(
         first: $first,
         skip: $skip,
         sortField: $sortField,
         sortOrder: $sortOrder,
         filter: $filter,
         display: $display
+        at: $at
         ) {
           edges {
             node {

src/utils/schema/resolvers/Query/all.ts

@@ -431,7 +431,7 @@ export default (entityName: string, fieldsByName: any, idsByName: any) =>
       if (skip || skip === 0) {
         const aggregation = await Record.aggregate([
           { $match: basicFilters },
-          ...(at ? getAtAggregation(at) : []),
+          ...(at ? getAtAggregation(new Date(at)) : []),
           ...linkedRecordsAggregation,
           ...linkedReferenceDataAggregation,
           ...defaultRecordAggregation,