Skip to content

Commit f07e887

Browse files
authored
Upgrade to Gremlin-Python 3.7 (#597)
* Upgrade to Gremlin-Python 3.7 * Add untyped TinkerPop serializers * update changelog
1 parent 6c0633c commit f07e887

File tree

9 files changed

+94
-82
lines changed

9 files changed

+94
-82
lines changed

ChangeLog.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ Starting with v1.31.6, this file will contain a record of major features and upd
55
## Upcoming
66
- Added `%reset_graph` line magic ([Link to PR](https://github.com/aws/graph-notebook/pull/610))
77
- Added `%get_graph` line magic and enabled `%status` for Neptune Analytics ([Link to PR](https://github.com/aws/graph-notebook/pull/611))
8+
- Upgraded to Gremlin-Python 3.7 ([Link to PR](https://github.com/aws/graph-notebook/pull/597))
89

910
## Release 4.3.1 (June 3, 2024)
1011

requirements.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@ jupyterlab_widgets>=1.0.0,<3.0.0
88
nbclient<=0.7.0
99
jupyter-contrib-nbextensions<=0.7.0
1010
widgetsnbextension<=3.6.1
11-
gremlinpython>=3.5.1,<=3.6.2
12-
requests>=2.32.0,<=2.32.2
11+
gremlinpython>=3.5.1,<=3.7.2
12+
requests>=2.27.0,<=2.31.0
1313
ipython>=7.16.1,<=8.10.0
1414
ipykernel==5.3.4
1515
ipyfilechooser==0.6.0

setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ def get_version():
6666
package_dir={'': 'src'},
6767
include_package_data=True,
6868
install_requires=[
69-
'gremlinpython>=3.5.1,<=3.6.2',
69+
'gremlinpython>=3.5.1,<=3.7.2',
7070
'SPARQLWrapper==2.0.0',
7171
'requests>=2.32.0,<=2.32.2',
7272
'ipywidgets==7.7.2',

src/graph_notebook/configuration/generate_config.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ def __init__(self, traversal_source: str = '', username: str = '', password: str
6666
traversal_source = DEFAULT_GREMLIN_TRAVERSAL_SOURCE
6767

6868
serializer_lower = message_serializer.lower()
69+
# TODO: Update with untyped serializers once supported in GremlinPython
6970
if serializer_lower == '':
7071
message_serializer = DEFAULT_GREMLIN_SERIALIZER
7172
elif serializer_lower in GRAPHSONV3_VARIANTS:

src/graph_notebook/magics/graph_magic.py

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -135,12 +135,16 @@
135135
SEED_NO_DATASETS_FOUND_MSG = "(No datasets available)"
136136
SEED_WIDGET_STYLE = {'description_width': '95px'}
137137

138+
# Tokens as currently defined in TinkerPop 3.7: https://github.com/apache/tinkerpop/blob/3.7-dev/gremlin-util/src/main/java/org/apache/tinkerpop/gremlin/util/ser/SerTokens.java
138139
serializers_map = {
139140
"MIME_JSON": "application/json",
140-
"GRAPHSON_V2D0": "application/vnd.gremlin-v2.0+json",
141-
"GRAPHSON_V3D0": "application/vnd.gremlin-v3.0+json",
142-
"GRYO_V3D0": "application/vnd.gremlin-v3.0+gryo",
143-
"GRAPHBINARY_V1D0": "application/vnd.graphbinary-v1.0"
141+
"GRAPHSON_V1": "application/vnd.gremlin-v1.0+json",
142+
"GRAPHSON_V1_UNTYPED": "application/vnd.gremlin-v1.0+json;types=false",
143+
"GRAPHSON_V2": "application/vnd.gremlin-v2.0+json",
144+
"GRAPHSON_V2_UNTYPED": "application/vnd.gremlin-v2.0+json;types=false",
145+
"GRAPHSON_V3": "application/vnd.gremlin-v3.0+json",
146+
"GRAPHSON_V3_UNTYPED": "application/vnd.gremlin-v3.0+json;types=false",
147+
"GRAPHBINARY_V1": "application/vnd.graphbinary-v1.0"
144148
}
145149

146150
DEFAULT_NAMEDGRAPH_URI = "http://aws.amazon.com/neptune/vocab/v01/DefaultNamedGraph"
@@ -1057,9 +1061,9 @@ def gremlin(self, line, cell, local_ns: dict = None):
10571061
'the profile report by default.')
10581062
parser.add_argument('--profile-chop', type=int, default=250,
10591063
help='Property to specify max length of profile results string. Default is 250')
1060-
parser.add_argument('--profile-serializer', type=str, default='application/json',
1064+
parser.add_argument('--profile-serializer', type=str, default='GRAPHSON_V3_UNTYPED',
10611065
help='Specify how to serialize results. Allowed values are any of the valid MIME type or '
1062-
'TinkerPop driver "Serializers" enum values. Default is application/json')
1066+
'TinkerPop driver "Serializers" enum values. Default is GRAPHSON_V3_UNTYPED')
10631067
parser.add_argument('--profile-indexOps', action='store_true', default=False,
10641068
help='Show a detailed report of all index operations.')
10651069
parser.add_argument('--profile-misc-args', type=str, default='{}',

src/graph_notebook/neptune/client.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -118,9 +118,9 @@
118118

119119
false_str_variants = [False, 'False', 'false', 'FALSE']
120120

121-
GRAPHSONV3_VARIANTS = ['graphsonv3', 'graphsonv3d0', 'graphsonserializersv3d0']
122-
GRAPHSONV2_VARIANTS = ['graphsonv2', 'graphsonv2d0', 'graphsonserializersv2d0']
123-
GRAPHBINARYV1_VARIANTS = ['graphbinaryv1', 'graphbinary', 'graphbinaryserializersv1']
121+
GRAPHSONV3_VARIANTS = ['graphsonv3', 'graphsonv3d0', 'graphsonserializersv3d0', 'graphsonmessageserializerv3']
122+
GRAPHSONV2_VARIANTS = ['graphsonv2', 'graphsonv2d0', 'graphsonserializersv2d0', 'graphsonmessageserializerv2']
123+
GRAPHBINARYV1_VARIANTS = ['graphbinaryv1', 'graphbinary', 'graphbinaryserializersv1', 'graphbinarymessageserializerv1']
124124

125125
STATISTICS_MODES = ["", "status", "disableAutoCompute", "enableAutoCompute", "refresh", "delete"]
126126
SUMMARY_MODES = ["", "basic", "detailed"]

test/unit/graph_magic/gremlin_profile_large_results_predicates.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,5 @@ Results
88
=======
99
Count: 999.999
1010
Output: [v[3], v[3600], v[3614], v[4], v[5], v[6], v[7], v[8], v[9], v[10], v[11], v[12], v[47], v[49], v[136], v[13], v[15], v[16], v[17], v[18], v[389], v[20], v[21], v[22], v[23], v[24], v[25], v[26], v[27], v[28], v[416], v[29], v[30], v[430], v[31], v[9...
11-
Response serializer: GRYO_V3D0
11+
Response serializer: application/vnd.gremlin-v3.0+json
1212
Response size (bytes): 23566

test/unit/graph_magic/gremlin_profile_sample_response.txt

Lines changed: 63 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -4,100 +4,106 @@
44

55
Query String
66
==================
7-
g.V().hasLabel("airport").has("code", "AUS").emit().repeat(in().simplePath()).times(2).limit(100)
7+
g.V().has('code','LHR').outE().inV().path().by(valueMap(true)).limit(5)
8+
89

910
Original Traversal
1011
==================
11-
[GraphStep(vertex,[]), HasStep([~label.eq(airport), code.eq(AUS)]), RepeatStep(emit(true),[VertexStep(IN,vertex), PathFilterStep(simple), RepeatEndStep],until(loops(2))), RangeGlobalStep(0,100)]
12+
[GraphStep(vertex,[]), HasStep([code.eq(LHR)]), VertexStep(OUT,edge), EdgeVertexStep(IN), PathStep([[PropertyMapStep(value)]]), RangeGlobalStep(0,5)]
1213

1314
Optimized Traversal
1415
===================
1516
Neptune steps:
1617
[
1718
NeptuneGraphQueryStep(Vertex) {
1819
JoinGroupNode {
19-
PatternNode[(?1, <code>, "AUS", ?) . project ?1 .], {estimatedCardinality=1, indexTime=84, hashJoin=true, joinTime=3, actualTotalOutput=1}
20-
PatternNode[(?1, <~label>, ?2=<airport>, <~>) . project ask .], {estimatedCardinality=3374, indexTime=29, hashJoin=true, joinTime=0, actualTotalOutput=61}
21-
RepeatNode {
22-
Repeat {
23-
PatternNode[(?3, ?5, ?1, ?6) . project ?1,?3 . IsEdgeIdFilter(?6) . SimplePathFilter(?1, ?3)) .], {hashJoin=true, estimatedCardinality=50148, indexTime=0, joinTime=3}
24-
}
25-
Emit {
26-
Filter(true)
27-
}
28-
LoopsCondition {
29-
LoopsFilter([?1, ?3],eq(2))
20+
PatternNode[VP(?1, <code>, "LHR", <~>) . project ?1 .], {estimatedCardinality=1, expectedTotalOutput=1, indexTime=0, joinTime=0, numSearches=1, actualTotalOutput=1}
21+
PatternNode[EL(?1, ?5, ?3, ?6) . project ?1,?6,?3 . IsEdgeIdFilter(?6) .], {estimatedCardinality=INFINITY, expectedTotalInput=1, indexTime=0, joinTime=0, numSearches=1}
22+
}, finishers=[limit(5)], {path=[Vertex(?1):GraphStep, Edge(?6,?1,?3):VertexStep, Vertex(?3):EdgeVertexStep], joinStats=true, optimizationTime=2, maxVarId=15, executionTime=14}
23+
},
24+
NeptunePathStep {
25+
NeptunePropertyMapStep {
26+
JoinGroupNode {
27+
PatternNode[VP(?1, ?8, ?9, <~>) .]
28+
}, {initialValues={?1=null, ?3=null, ?6=null}}
29+
},
30+
NeptunePropertyMapStep {
31+
JoinGroupNode {
32+
UnionNode {
33+
PatternNode[EP(?6, ?10, ?11, <~>) .], {estimatedCardinality=63580}
34+
PatternNode[EL(?, ?12, ?, ?6) .], {estimatedCardinality=INFINITY}
3035
}
31-
}, annotations={repeatMode=BFS, emitFirst=true, untilFirst=false, leftVar=?1, rightVar=?3}
32-
}, finishers=[limit(100)], annotations={path=[Vertex(?1):GraphStep, Repeat[Vertex(?3):VertexStep]], joinStats=true, optimizationTime=495, maxVarId=7, executionTime=323}
36+
}, {initialValues={?1=null, ?3=null, ?6=null}}
37+
},
38+
NeptunePropertyMapStep {
39+
JoinGroupNode {
40+
PatternNode[VP(?3, ?13, ?14, <~>) .]
41+
}, {initialValues={?1=null, ?3=null, ?6=null}}
42+
}
3343
},
3444
NeptuneTraverserConverterStep
3545
]
3646

47+
3748
Physical Pipeline
3849
=================
3950
NeptuneGraphQueryStep
4051
|-- StartOp
41-
|-- JoinGroupOp
42-
|-- SpoolerOp(100)
43-
|-- DynamicJoinOp(PatternNode[(?1, <code>, "AUS", ?) . project ?1 .], {estimatedCardinality=1, indexTime=84, hashJoin=true})
44-
|-- SpoolerOp(100)
45-
|-- DynamicJoinOp(PatternNode[(?1, <~label>, ?2=<airport>, <~>) . project ask .], {estimatedCardinality=3374, indexTime=29, hashJoin=true})
46-
|-- RepeatOp
47-
|-- <upstream input> (Iteration 0) [visited=1, output=1 (until=0, emit=1), next=1]
48-
|-- BindingSetQueue (Iteration 1) [visited=61, output=61 (until=0, emit=61), next=61]
49-
|-- SpoolerOp(100)
50-
|-- DynamicJoinOp(PatternNode[(?3, ?5, ?1, ?6) . project ?1,?3 . IsEdgeIdFilter(?6) . SimplePathFilter(?1, ?3)) .], {hashJoin=true, estimatedCardinality=50148, indexTime=0})
51-
|-- BindingSetQueue (Iteration 2) [visited=38, output=38 (until=38, emit=0), next=0]
52-
|-- SpoolerOp(100)
53-
|-- DynamicJoinOp(PatternNode[(?3, ?5, ?1, ?6) . project ?1,?3 . IsEdgeIdFilter(?6) . SimplePathFilter(?1, ?3)) .], {hashJoin=true, estimatedCardinality=50148, indexTime=0})
54-
|-- LimitOp(100)
52+
|-- JoinGroupOp@2f500db7
53+
|-- DynamicJoinOp@78b965ae(PipelineJoinOp(PatternNode[VP(?1, <code>, "LHR", <~>) . project ?1 .], {estimatedCardinality=1, expectedTotalOutput=1}))
54+
|-- SpoolerOp(100, @78b965ae, null)
55+
|-- DynamicJoinOp@67012ec9(PipelineJoinOp(PatternNode[EL(?1, ?5, ?3, ?6) . project ?1,?6,?3 . IsEdgeIdFilter(?6) .], {estimatedCardinality=INFINITY, expectedTotalInput=1}))
56+
|-- LimitOp(5)
57+
58+
NeptunePathStep
59+
|-- BindingSetQueue
60+
|-- JoinGroupOp@6a996745
61+
|-- DynamicJoinOp@417c3e4f(PipelineJoinOp(PatternNode[VP(?1, ?8, ?9, <~>) .]))
62+
63+
|-- BindingSetQueue
64+
|-- JoinGroupOp@2b0c033
65+
|-- UnionOp@44764c1
66+
|-- MultiplexerOp
67+
68+
|-- BindingSetQueue
69+
|-- JoinGroupOp@60bf88a9
70+
|-- DynamicJoinOp@32903d93(PipelineJoinOp(PatternNode[VP(?3, ?13, ?14, <~>) .]))
5571

5672
Runtime (ms)
5773
============
58-
Query Execution: 392.686
59-
Serialization: 2636.380
74+
Query Execution: 18.669
75+
Serialization: 15.464
6076

6177
Traversal Metrics
6278
=================
6379
Step Count Traversers Time (ms) % Dur
6480
-------------------------------------------------------------------------------------------------------------
65-
NeptuneGraphQueryStep(Vertex) 100 100 314.162 82.78
66-
NeptuneTraverserConverterStep 100 100 65.333 17.22
67-
>TOTAL - - 379.495 -
68-
69-
Repeat Metrics
70-
==============
71-
Iteration Visited Output Until Emit Next
72-
------------------------------------------------------
73-
0 1 1 0 1 1
74-
1 61 61 0 61 61
75-
2 38 38 38 0 0
76-
------------------------------------------------------
77-
100 100 38 62 62
81+
NeptuneGraphQueryStep(Vertex) 5 5 2.152 20.56
82+
NeptunePathStep([[NeptunePropertyMapStep], [Nep... 5 5 4.317 41.25
83+
NeptuneTraverserConverterStep 5 5 3.997 38.19
84+
>TOTAL - - 10.466 -
7885

7986
Predicates
8087
==========
81-
# of predicates: 16
82-
83-
WARNING: reverse traversal with no edge label(s) - .in() / .both() may impact query performance
88+
# of predicates: 18
8489

8590
Results
8691
=======
87-
Count: 100
88-
Output: [v[3], v[3600], v[3614], v[4], v[5], v[6], v[7], v[8], v[9], v[10], v[11], v[12], v[47], v[49], v[136], v[13], v[15], v[16], v[17], v[18], v[389], v[20], v[21], v[22], v[23], v[24], v[25], v[26], v[27], v[28], v[416], v[29], v[30], v[430], v[31], v[9...
89-
Response serializer: GRYO_V3D0
90-
Response size (bytes): 23566
92+
Count: 5
93+
Output: [path[{country=[UK], code=[LHR], longest=[12799], city=[London], lon=[-0.461941003799], type=[airport], label=airport, elev=[83], icao=[EGLL], id=49, runways=[2], region=[GB-ENG], lat=[51.4706001282], desc=[London Heathrow]}, {dist=3533, id=9144, lab...
94+
Response serializer: application/vnd.gremlin-v3.0+json
95+
Response size (bytes): 10162
96+
9197

9298
Index Operations
9399
================
94100
Query execution:
95-
# of statement index ops: 3
96-
# of unique statement index ops: 3
101+
# of statement index ops: 18
102+
# of unique statement index ops: 18
97103
Duplication ratio: 1.0
98104
# of terms materialized: 0
99105
Serialization:
100-
# of statement index ops: 200
101-
# of unique statement index ops: 140
102-
Duplication ratio: 1.43
103-
# of terms materialized: 393
106+
# of statement index ops: 18
107+
# of unique statement index ops: 18
108+
Duplication ratio: 1.0
109+
# of terms materialized: 0

test/unit/graph_magic/metadata_gremlin_profile.py

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,20 +11,20 @@
1111
class TestMetadataClassFunctions(unittest.TestCase):
1212

1313
def test_gremlin_profile_metadata_func(self):
14-
time_expected = 392.686
15-
predicates_expected = 16
16-
results_num_expected = 100
17-
serialization_expected = 2636.380
18-
serializer_type_expected = "GRYO_V3D0"
19-
results_size_expected = 23566
20-
query_total_index_ops_expected = 3
21-
query_unique_index_ops_expected = 3
14+
time_expected = 18.669
15+
predicates_expected = 18
16+
results_num_expected = 5
17+
serialization_expected = 15.464
18+
serializer_type_expected = "application/vnd.gremlin-v3.0+json"
19+
results_size_expected = 10162
20+
query_total_index_ops_expected = 18
21+
query_unique_index_ops_expected = 18
2222
query_duplication_ratio_expected = 1
2323
query_terms_materialized_expected = 0
24-
seri_total_index_ops_expected = 200
25-
seri_unique_index_ops_expected = 140
26-
seri_duplication_ratio_expected = 1.43
27-
seri_terms_materialized_expected = 393
24+
seri_total_index_ops_expected = 18
25+
seri_unique_index_ops_expected = 18
26+
seri_duplication_ratio_expected = 1.0
27+
seri_terms_materialized_expected = 0
2828

2929
gremlin_metadata = Metadata()
3030
with open('gremlin_profile_sample_response.txt', 'r') as profile_file:

0 commit comments

Comments
 (0)