D4488: wireprotov2: define and implement "manifestdata" command
This revision was automatically updated to reflect the committed changes.
Closed by commit rHGc7a7c7e844e5: wireprotov2: define and implement
manifestdata command (authored by indygreg, committed by ).
REPOSITORY
rHG Mercurial
CHANGES SINCE LAST UPDATE
https://phab.mercurial-scm.org/D4488?vs=10967=11066
REVISION DETAIL
https://phab.mercurial-scm.org/D4488
AFFECTED FILES
mercurial/help/internals/wireprotocolv2.txt
mercurial/wireprotov2server.py
tests/test-http-protocol.t
tests/test-wireproto-command-capabilities.t
tests/test-wireproto-command-manifestdata.t
CHANGE DETAILS
diff --git a/tests/test-wireproto-command-manifestdata.t
b/tests/test-wireproto-command-manifestdata.t
new file mode 100644
--- /dev/null
+++ b/tests/test-wireproto-command-manifestdata.t
@@ -0,0 +1,507 @@
+ $ . $TESTDIR/wireprotohelpers.sh
+
+ $ hg init server
+ $ enablehttpv2 server
+ $ cd server
+ $ echo a0 > a
+ $ echo b0 > b
+ $ mkdir -p dir0/child0 dir0/child1 dir1
+ $ echo c0 > dir0/c
+ $ echo d0 > dir0/d
+ $ echo e0 > dir0/child0/e
+ $ echo f0 > dir0/child1/f
+ $ hg -q commit -A -m 'commit 0'
+
+ $ echo a1 > a
+ $ echo d1 > dir0/d
+ $ hg commit -m 'commit 1'
+ $ echo f0 > dir0/child1/f
+ $ hg commit -m 'commit 2'
+ nothing changed
+ [1]
+
+ $ hg -q up -r 0
+ $ echo a2 > a
+ $ hg commit -m 'commit 3'
+ created new head
+
+ $ hg log -G -T '{rev}:{node} {desc}\n'
+ @ 2:c8757a2ffe552850d1e0dfe60d295ebf64c196d9 commit 3
+ |
+ | o 1:650165e803375748a94df471e5b58d85763e0b29 commit 1
+ |/
+ o 0:6d85ca1270b377d320098556ba5bfad34a9ee12d commit 0
+
+
+ $ hg --debug debugindex -m
+ rev linkrev nodeid p1
p2
+ 0 0 1b175b595f022cfab5b809cc0ed551bd0b3ff5e4
+ 1 1 91e0bdbfb0dde0023fa063edc1445f207a22eac7
1b175b595f022cfab5b809cc0ed551bd0b3ff5e4
+ 2 2 46a6721b5edaf0ea04b79a5cb3218854a4d2aba0
1b175b595f022cfab5b809cc0ed551bd0b3ff5e4
+
+ $ hg serve -p $HGPORT -d --pid-file hg.pid -E error.log
+ $ cat hg.pid > $DAEMON_PIDS
+
+Missing arguments is an error
+
+ $ sendhttpv2peer << EOF
+ > command manifestdata
+ > EOF
+ creating http peer for wire protocol version 2
+ sending manifestdata command
+ s> POST /api/exp-http-v2-0001/ro/manifestdata HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> accept: application/mercurial-exp-framing-0005\r\n
+ s> content-type: application/mercurial-exp-framing-0005\r\n
+ s> content-length: 27\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s> user-agent: Mercurial debugwireproto\r\n
+ s> \r\n
+ s> \x13\x00\x00\x01\x00\x01\x01\x11\xa1DnameLmanifestdata
+ s> makefile('rb', None)
+ s> HTTP/1.1 200 OK\r\n
+ s> Server: testing stub value\r\n
+ s> Date: $HTTP_DATE$\r\n
+ s> Content-Type: application/mercurial-exp-framing-0005\r\n
+ s> Transfer-Encoding: chunked\r\n
+ s> \r\n
+ s> 45\r\n
+ s> =\x00\x00\x01\x00\x02\x012
+ s> \xa2Eerror\xa1GmessageX\x1enodes argument must be definedFstatusEerror
+ s> \r\n
+ received frame(size=61; request=1; stream=2; streamflags=stream-begin;
type=command-response; flags=eos)
+ s> 0\r\n
+ s> \r\n
+ abort: nodes argument must be defined!
+ [255]
+
+ $ sendhttpv2peer << EOF
+ > command manifestdata
+ > nodes eval:[]
+ > EOF
+ creating http peer for wire protocol version 2
+ sending manifestdata command
+ s> POST /api/exp-http-v2-0001/ro/manifestdata HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> accept: application/mercurial-exp-framing-0005\r\n
+ s> content-type: application/mercurial-exp-framing-0005\r\n
+ s> content-length: 40\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s> user-agent: Mercurial debugwireproto\r\n
+ s> \r\n
+ s> \x00\x00\x01\x00\x01\x01\x11\xa2Dargs\xa1Enodes\x80DnameLmanifestdata
+ s> makefile('rb', None)
+ s> HTTP/1.1 200 OK\r\n
+ s> Server: testing stub value\r\n
+ s> Date: $HTTP_DATE$\r\n
+ s> Content-Type: application/mercurial-exp-framing-0005\r\n
+ s> Transfer-Encoding: chunked\r\n
+ s> \r\n
+ s> 44\r\n
+ s> <\x00\x00\x01\x00\x02\x012
+ s> \xa2Eerror\xa1GmessageX\x1dtree argument must be definedFstatusEerror
+ s> \r\n
+ received frame(size=60; request=1; stream=2; streamflags=stream-begin;
type=command-response; flags=eos)
+ s> 0\r\n
+ s> \r\n
+ abort: tree argument must be defined!
+ [255]
+
+Unknown node is an error
+
+ $ sendhttpv2peer << EOF
+ > command manifestdata
+ > nodes
eval:[b'\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa']
+ > tree eval:b''
+ > EOF
+ creating http peer for wire protocol version 2
+ sending manifestdata command
+ s>
D4488: wireprotov2: define and implement "manifestdata" command
indygreg updated this revision to Diff 10967.
REPOSITORY
rHG Mercurial
CHANGES SINCE LAST UPDATE
https://phab.mercurial-scm.org/D4488?vs=10801=10967
REVISION DETAIL
https://phab.mercurial-scm.org/D4488
AFFECTED FILES
mercurial/help/internals/wireprotocolv2.txt
mercurial/wireprotov2server.py
tests/test-http-protocol.t
tests/test-wireproto-command-capabilities.t
tests/test-wireproto-command-manifestdata.t
CHANGE DETAILS
diff --git a/tests/test-wireproto-command-manifestdata.t
b/tests/test-wireproto-command-manifestdata.t
new file mode 100644
--- /dev/null
+++ b/tests/test-wireproto-command-manifestdata.t
@@ -0,0 +1,507 @@
+ $ . $TESTDIR/wireprotohelpers.sh
+
+ $ hg init server
+ $ enablehttpv2 server
+ $ cd server
+ $ echo a0 > a
+ $ echo b0 > b
+ $ mkdir -p dir0/child0 dir0/child1 dir1
+ $ echo c0 > dir0/c
+ $ echo d0 > dir0/d
+ $ echo e0 > dir0/child0/e
+ $ echo f0 > dir0/child1/f
+ $ hg -q commit -A -m 'commit 0'
+
+ $ echo a1 > a
+ $ echo d1 > dir0/d
+ $ hg commit -m 'commit 1'
+ $ echo f0 > dir0/child1/f
+ $ hg commit -m 'commit 2'
+ nothing changed
+ [1]
+
+ $ hg -q up -r 0
+ $ echo a2 > a
+ $ hg commit -m 'commit 3'
+ created new head
+
+ $ hg log -G -T '{rev}:{node} {desc}\n'
+ @ 2:c8757a2ffe552850d1e0dfe60d295ebf64c196d9 commit 3
+ |
+ | o 1:650165e803375748a94df471e5b58d85763e0b29 commit 1
+ |/
+ o 0:6d85ca1270b377d320098556ba5bfad34a9ee12d commit 0
+
+
+ $ hg --debug debugindex -m
+ rev linkrev nodeid p1
p2
+ 0 0 1b175b595f022cfab5b809cc0ed551bd0b3ff5e4
+ 1 1 91e0bdbfb0dde0023fa063edc1445f207a22eac7
1b175b595f022cfab5b809cc0ed551bd0b3ff5e4
+ 2 2 46a6721b5edaf0ea04b79a5cb3218854a4d2aba0
1b175b595f022cfab5b809cc0ed551bd0b3ff5e4
+
+ $ hg serve -p $HGPORT -d --pid-file hg.pid -E error.log
+ $ cat hg.pid > $DAEMON_PIDS
+
+Missing arguments is an error
+
+ $ sendhttpv2peer << EOF
+ > command manifestdata
+ > EOF
+ creating http peer for wire protocol version 2
+ sending manifestdata command
+ s> POST /api/exp-http-v2-0001/ro/manifestdata HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> accept: application/mercurial-exp-framing-0005\r\n
+ s> content-type: application/mercurial-exp-framing-0005\r\n
+ s> content-length: 27\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s> user-agent: Mercurial debugwireproto\r\n
+ s> \r\n
+ s> \x13\x00\x00\x01\x00\x01\x01\x11\xa1DnameLmanifestdata
+ s> makefile('rb', None)
+ s> HTTP/1.1 200 OK\r\n
+ s> Server: testing stub value\r\n
+ s> Date: $HTTP_DATE$\r\n
+ s> Content-Type: application/mercurial-exp-framing-0005\r\n
+ s> Transfer-Encoding: chunked\r\n
+ s> \r\n
+ s> 45\r\n
+ s> =\x00\x00\x01\x00\x02\x012
+ s> \xa2Eerror\xa1GmessageX\x1enodes argument must be definedFstatusEerror
+ s> \r\n
+ received frame(size=61; request=1; stream=2; streamflags=stream-begin;
type=command-response; flags=eos)
+ s> 0\r\n
+ s> \r\n
+ abort: nodes argument must be defined!
+ [255]
+
+ $ sendhttpv2peer << EOF
+ > command manifestdata
+ > nodes eval:[]
+ > EOF
+ creating http peer for wire protocol version 2
+ sending manifestdata command
+ s> POST /api/exp-http-v2-0001/ro/manifestdata HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> accept: application/mercurial-exp-framing-0005\r\n
+ s> content-type: application/mercurial-exp-framing-0005\r\n
+ s> content-length: 40\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s> user-agent: Mercurial debugwireproto\r\n
+ s> \r\n
+ s> \x00\x00\x01\x00\x01\x01\x11\xa2Dargs\xa1Enodes\x80DnameLmanifestdata
+ s> makefile('rb', None)
+ s> HTTP/1.1 200 OK\r\n
+ s> Server: testing stub value\r\n
+ s> Date: $HTTP_DATE$\r\n
+ s> Content-Type: application/mercurial-exp-framing-0005\r\n
+ s> Transfer-Encoding: chunked\r\n
+ s> \r\n
+ s> 44\r\n
+ s> <\x00\x00\x01\x00\x02\x012
+ s> \xa2Eerror\xa1GmessageX\x1dtree argument must be definedFstatusEerror
+ s> \r\n
+ received frame(size=60; request=1; stream=2; streamflags=stream-begin;
type=command-response; flags=eos)
+ s> 0\r\n
+ s> \r\n
+ abort: tree argument must be defined!
+ [255]
+
+Unknown node is an error
+
+ $ sendhttpv2peer << EOF
+ > command manifestdata
+ > nodes
eval:[b'\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa']
+ > tree eval:b''
+ > EOF
+ creating http peer for wire protocol version 2
+ sending manifestdata command
+ s> POST /api/exp-http-v2-0001/ro/manifestdata HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> accept:
D4488: wireprotov2: define and implement "manifestdata" command
indygreg added a comment. In https://phab.mercurial-scm.org/D4488#69449, @durin42 wrote: > Could you add a TODO about the overhead of having to enumerate every node required? That'll be prohibitive fairly quickly I think. There are already TODOs in wireprotocolv2.txt. And, yes, the overhead is prohibitive fairly quickly and I plan to add more "bulk querying" capabilities in future commits. I'm trying to get the granular data access in first then optimize later. REPOSITORY rHG Mercurial REVISION DETAIL https://phab.mercurial-scm.org/D4488 To: indygreg, #hg-reviewers Cc: durin42, mercurial-devel ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
D4488: wireprotov2: define and implement "manifestdata" command
durin42 added a comment. Could you add a TODO about the overhead of having to enumerate every node required? That'll be prohibitive fairly quickly I think. REPOSITORY rHG Mercurial REVISION DETAIL https://phab.mercurial-scm.org/D4488 To: indygreg, #hg-reviewers Cc: durin42, mercurial-devel ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
D4488: wireprotov2: define and implement "manifestdata" command
indygreg created this revision.
Herald added a subscriber: mercurial-devel.
Herald added a reviewer: hg-reviewers.
REVISION SUMMARY
The added command can be used for obtaining manifest data.
Given a manifest path and set of manifest nodes, data about
manifests can be retrieved.
Unlike changeset data, we wish to emit deltas to describe
manifest revisions. So the command uses the relatively new
API for building delta requests and emitting them.
The code calls into deltaparent(), which I'm not very keen of.
There's still work to be done in delta generation land so
implementation details of storage (e.g. exactly one delta
is stored/available) don't creep into higher levels. But we
can worry about this later (there is already a TODO on
imanifestorage tracking this).
On the subject of parent deltas, the server assumes parent revisions
exist on the receiving end. This is obviously wrong for shallow
clone. I've added TODOs to add a mechanism to the command to
allow clients to specify desired behavior. This shouldn't be
too difficult to implement.
Another big change is that the client must explicitly request
manifest nodes to retrieve. This is a major departure from
"getbundle," where the server derives relevant manifests as it
iterates changesets and sends them automatically. As implemented,
the client must transmit each requested node to the server. At
20 bytes per node, we're looking at 2 MB per 100,000 nodes. Plus
wire encoding overhead. This isn't ideal for clients with limited
upload bandwidth. I plan to address this in the future by allowing
alternate mechanisms for defining the revisions to retrieve. One
idea is to define a range of changeset revisions whose manifest
revisions to retrieve (similar to how "changesetdata" works).
We almost certainly want an API to look up an individual manifest
by node. And that's where I've chosen to start with the implementation.
Again, a theme of this early exchangev2 work is I want to start by
building primitives for accessing raw repository data first and see
how far we can get with those before we need more complexity.
REPOSITORY
rHG Mercurial
REVISION DETAIL
https://phab.mercurial-scm.org/D4488
AFFECTED FILES
mercurial/help/internals/wireprotocolv2.txt
mercurial/wireprotov2server.py
tests/test-http-protocol.t
tests/test-wireproto-command-capabilities.t
tests/test-wireproto-command-manifestdata.t
CHANGE DETAILS
diff --git a/tests/test-wireproto-command-manifestdata.t
b/tests/test-wireproto-command-manifestdata.t
new file mode 100644
--- /dev/null
+++ b/tests/test-wireproto-command-manifestdata.t
@@ -0,0 +1,507 @@
+ $ . $TESTDIR/wireprotohelpers.sh
+
+ $ hg init server
+ $ enablehttpv2 server
+ $ cd server
+ $ echo a0 > a
+ $ echo b0 > b
+ $ mkdir -p dir0/child0 dir0/child1 dir1
+ $ echo c0 > dir0/c
+ $ echo d0 > dir0/d
+ $ echo e0 > dir0/child0/e
+ $ echo f0 > dir0/child1/f
+ $ hg -q commit -A -m 'commit 0'
+
+ $ echo a1 > a
+ $ echo d1 > dir0/d
+ $ hg commit -m 'commit 1'
+ $ echo f0 > dir0/child1/f
+ $ hg commit -m 'commit 2'
+ nothing changed
+ [1]
+
+ $ hg -q up -r 0
+ $ echo a2 > a
+ $ hg commit -m 'commit 3'
+ created new head
+
+ $ hg log -G -T '{rev}:{node} {desc}\n'
+ @ 2:c8757a2ffe552850d1e0dfe60d295ebf64c196d9 commit 3
+ |
+ | o 1:650165e803375748a94df471e5b58d85763e0b29 commit 1
+ |/
+ o 0:6d85ca1270b377d320098556ba5bfad34a9ee12d commit 0
+
+
+ $ hg --debug debugindex -m
+ rev linkrev nodeid p1
p2
+ 0 0 1b175b595f022cfab5b809cc0ed551bd0b3ff5e4
+ 1 1 91e0bdbfb0dde0023fa063edc1445f207a22eac7
1b175b595f022cfab5b809cc0ed551bd0b3ff5e4
+ 2 2 46a6721b5edaf0ea04b79a5cb3218854a4d2aba0
1b175b595f022cfab5b809cc0ed551bd0b3ff5e4
+
+ $ hg serve -p $HGPORT -d --pid-file hg.pid -E error.log
+ $ cat hg.pid > $DAEMON_PIDS
+
+Missing arguments is an error
+
+ $ sendhttpv2peer << EOF
+ > command manifestdata
+ > EOF
+ creating http peer for wire protocol version 2
+ sending manifestdata command
+ s> POST /api/exp-http-v2-0001/ro/manifestdata HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> accept: application/mercurial-exp-framing-0005\r\n
+ s> content-type: application/mercurial-exp-framing-0005\r\n
+ s> content-length: 27\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s> user-agent: Mercurial debugwireproto\r\n
+ s> \r\n
+ s> \x13\x00\x00\x01\x00\x01\x01\x11\xa1DnameLmanifestdata
+ s> makefile('rb', None)
+ s> HTTP/1.1 200 OK\r\n
+ s> Server: testing stub value\r\n
+ s> Date: $HTTP_DATE$\r\n
+ s> Content-Type: application/mercurial-exp-framing-0005\r\n
+ s> Transfer-Encoding: chunked\r\n
