D4490: wireprotov2: define and implement "filedata" command
This revision was automatically updated to reflect the committed changes.
Closed by commit rHG0e03e6a44dee: wireprotov2: define and implement
filedata command (authored by indygreg, committed by ).
REPOSITORY
rHG Mercurial
CHANGES SINCE LAST UPDATE
https://phab.mercurial-scm.org/D4490?vs=10969=11068
REVISION DETAIL
https://phab.mercurial-scm.org/D4490
AFFECTED FILES
mercurial/help/internals/wireprotocolv2.txt
mercurial/wireprotov2server.py
tests/test-http-protocol.t
tests/test-wireproto-command-capabilities.t
tests/test-wireproto-command-filedata.t
CHANGE DETAILS
diff --git a/tests/test-wireproto-command-filedata.t
b/tests/test-wireproto-command-filedata.t
new file mode 100644
--- /dev/null
+++ b/tests/test-wireproto-command-filedata.t
@@ -0,0 +1,497 @@
+ $ . $TESTDIR/wireprotohelpers.sh
+
+ $ hg init server
+ $ enablehttpv2 server
+ $ cd server
+ $ echo a0 > a
+ $ echo b0 > b
+ $ mkdir -p dir0/child0 dir0/child1 dir1
+ $ echo c0 > dir0/c
+ $ echo d0 > dir0/d
+ $ echo e0 > dir0/child0/e
+ $ echo f0 > dir0/child1/f
+ $ hg -q commit -A -m 'commit 0'
+
+ $ echo a1 > a
+ $ echo d1 > dir0/d
+ $ hg commit -m 'commit 1'
+ $ echo f0 > dir0/child1/f
+ $ hg commit -m 'commit 2'
+ nothing changed
+ [1]
+
+ $ hg -q up -r 0
+ $ echo a2 > a
+ $ hg commit -m 'commit 3'
+ created new head
+
+ $ hg log -G -T '{rev}:{node} {desc}\n'
+ @ 2:c8757a2ffe552850d1e0dfe60d295ebf64c196d9 commit 3
+ |
+ | o 1:650165e803375748a94df471e5b58d85763e0b29 commit 1
+ |/
+ o 0:6d85ca1270b377d320098556ba5bfad34a9ee12d commit 0
+
+
+ $ hg --debug debugindex a
+ rev linkrev nodeid p1
p2
+ 0 0 2b4eb07319bfa077a40a2f04913659aef0da42da
+ 1 1 9a38122997b3ac97be2a9aa2e556838341fdf2cc
2b4eb07319bfa077a40a2f04913659aef0da42da
+ 2 2 0879345e39377229634b420c639454156726c6b6
2b4eb07319bfa077a40a2f04913659aef0da42da
+
+ $ hg --debug debugindex dir0/child0/e
+ rev linkrev nodeid p1
p2
+ 0 0 bbba6c06b30f443d34ff841bc985c4d0827c6be4
+
+ $ hg serve -p $HGPORT -d --pid-file hg.pid -E error.log
+ $ cat hg.pid > $DAEMON_PIDS
+
+Missing arguments is an error
+
+ $ sendhttpv2peer << EOF
+ > command filedata
+ > EOF
+ creating http peer for wire protocol version 2
+ sending filedata command
+ s> POST /api/exp-http-v2-0001/ro/filedata HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> accept: application/mercurial-exp-framing-0005\r\n
+ s> content-type: application/mercurial-exp-framing-0005\r\n
+ s> content-length: 23\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s> user-agent: Mercurial debugwireproto\r\n
+ s> \r\n
+ s> \x0f\x00\x00\x01\x00\x01\x01\x11\xa1DnameHfiledata
+ s> makefile('rb', None)
+ s> HTTP/1.1 200 OK\r\n
+ s> Server: testing stub value\r\n
+ s> Date: $HTTP_DATE$\r\n
+ s> Content-Type: application/mercurial-exp-framing-0005\r\n
+ s> Transfer-Encoding: chunked\r\n
+ s> \r\n
+ s> 45\r\n
+ s> =\x00\x00\x01\x00\x02\x012
+ s> \xa2Eerror\xa1GmessageX\x1enodes argument must be definedFstatusEerror
+ s> \r\n
+ received frame(size=61; request=1; stream=2; streamflags=stream-begin;
type=command-response; flags=eos)
+ s> 0\r\n
+ s> \r\n
+ abort: nodes argument must be defined!
+ [255]
+
+ $ sendhttpv2peer << EOF
+ > command filedata
+ > nodes eval:[]
+ > EOF
+ creating http peer for wire protocol version 2
+ sending filedata command
+ s> POST /api/exp-http-v2-0001/ro/filedata HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> accept: application/mercurial-exp-framing-0005\r\n
+ s> content-type: application/mercurial-exp-framing-0005\r\n
+ s> content-length: 36\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s> user-agent: Mercurial debugwireproto\r\n
+ s> \r\n
+ s> \x1c\x00\x00\x01\x00\x01\x01\x11\xa2Dargs\xa1Enodes\x80DnameHfiledata
+ s> makefile('rb', None)
+ s> HTTP/1.1 200 OK\r\n
+ s> Server: testing stub value\r\n
+ s> Date: $HTTP_DATE$\r\n
+ s> Content-Type: application/mercurial-exp-framing-0005\r\n
+ s> Transfer-Encoding: chunked\r\n
+ s> \r\n
+ s> 44\r\n
+ s> <\x00\x00\x01\x00\x02\x012
+ s> \xa2Eerror\xa1GmessageX\x1dpath argument must be definedFstatusEerror
+ s> \r\n
+ received frame(size=60; request=1; stream=2; streamflags=stream-begin;
type=command-response; flags=eos)
+ s> 0\r\n
+ s> \r\n
+ abort: path argument must be defined!
+ [255]
+
+Unknown node is an error
+
+ $ sendhttpv2peer << EOF
+ > command
D4490: wireprotov2: define and implement "filedata" command
indygreg updated this revision to Diff 10969.
REPOSITORY
rHG Mercurial
CHANGES SINCE LAST UPDATE
https://phab.mercurial-scm.org/D4490?vs=10803=10969
REVISION DETAIL
https://phab.mercurial-scm.org/D4490
AFFECTED FILES
mercurial/help/internals/wireprotocolv2.txt
mercurial/wireprotov2server.py
tests/test-http-protocol.t
tests/test-wireproto-command-capabilities.t
tests/test-wireproto-command-filedata.t
CHANGE DETAILS
diff --git a/tests/test-wireproto-command-filedata.t
b/tests/test-wireproto-command-filedata.t
new file mode 100644
--- /dev/null
+++ b/tests/test-wireproto-command-filedata.t
@@ -0,0 +1,497 @@
+ $ . $TESTDIR/wireprotohelpers.sh
+
+ $ hg init server
+ $ enablehttpv2 server
+ $ cd server
+ $ echo a0 > a
+ $ echo b0 > b
+ $ mkdir -p dir0/child0 dir0/child1 dir1
+ $ echo c0 > dir0/c
+ $ echo d0 > dir0/d
+ $ echo e0 > dir0/child0/e
+ $ echo f0 > dir0/child1/f
+ $ hg -q commit -A -m 'commit 0'
+
+ $ echo a1 > a
+ $ echo d1 > dir0/d
+ $ hg commit -m 'commit 1'
+ $ echo f0 > dir0/child1/f
+ $ hg commit -m 'commit 2'
+ nothing changed
+ [1]
+
+ $ hg -q up -r 0
+ $ echo a2 > a
+ $ hg commit -m 'commit 3'
+ created new head
+
+ $ hg log -G -T '{rev}:{node} {desc}\n'
+ @ 2:c8757a2ffe552850d1e0dfe60d295ebf64c196d9 commit 3
+ |
+ | o 1:650165e803375748a94df471e5b58d85763e0b29 commit 1
+ |/
+ o 0:6d85ca1270b377d320098556ba5bfad34a9ee12d commit 0
+
+
+ $ hg --debug debugindex a
+ rev linkrev nodeid p1
p2
+ 0 0 2b4eb07319bfa077a40a2f04913659aef0da42da
+ 1 1 9a38122997b3ac97be2a9aa2e556838341fdf2cc
2b4eb07319bfa077a40a2f04913659aef0da42da
+ 2 2 0879345e39377229634b420c639454156726c6b6
2b4eb07319bfa077a40a2f04913659aef0da42da
+
+ $ hg --debug debugindex dir0/child0/e
+ rev linkrev nodeid p1
p2
+ 0 0 bbba6c06b30f443d34ff841bc985c4d0827c6be4
+
+ $ hg serve -p $HGPORT -d --pid-file hg.pid -E error.log
+ $ cat hg.pid > $DAEMON_PIDS
+
+Missing arguments is an error
+
+ $ sendhttpv2peer << EOF
+ > command filedata
+ > EOF
+ creating http peer for wire protocol version 2
+ sending filedata command
+ s> POST /api/exp-http-v2-0001/ro/filedata HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> accept: application/mercurial-exp-framing-0005\r\n
+ s> content-type: application/mercurial-exp-framing-0005\r\n
+ s> content-length: 23\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s> user-agent: Mercurial debugwireproto\r\n
+ s> \r\n
+ s> \x0f\x00\x00\x01\x00\x01\x01\x11\xa1DnameHfiledata
+ s> makefile('rb', None)
+ s> HTTP/1.1 200 OK\r\n
+ s> Server: testing stub value\r\n
+ s> Date: $HTTP_DATE$\r\n
+ s> Content-Type: application/mercurial-exp-framing-0005\r\n
+ s> Transfer-Encoding: chunked\r\n
+ s> \r\n
+ s> 45\r\n
+ s> =\x00\x00\x01\x00\x02\x012
+ s> \xa2Eerror\xa1GmessageX\x1enodes argument must be definedFstatusEerror
+ s> \r\n
+ received frame(size=61; request=1; stream=2; streamflags=stream-begin;
type=command-response; flags=eos)
+ s> 0\r\n
+ s> \r\n
+ abort: nodes argument must be defined!
+ [255]
+
+ $ sendhttpv2peer << EOF
+ > command filedata
+ > nodes eval:[]
+ > EOF
+ creating http peer for wire protocol version 2
+ sending filedata command
+ s> POST /api/exp-http-v2-0001/ro/filedata HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> accept: application/mercurial-exp-framing-0005\r\n
+ s> content-type: application/mercurial-exp-framing-0005\r\n
+ s> content-length: 36\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s> user-agent: Mercurial debugwireproto\r\n
+ s> \r\n
+ s> \x1c\x00\x00\x01\x00\x01\x01\x11\xa2Dargs\xa1Enodes\x80DnameHfiledata
+ s> makefile('rb', None)
+ s> HTTP/1.1 200 OK\r\n
+ s> Server: testing stub value\r\n
+ s> Date: $HTTP_DATE$\r\n
+ s> Content-Type: application/mercurial-exp-framing-0005\r\n
+ s> Transfer-Encoding: chunked\r\n
+ s> \r\n
+ s> 44\r\n
+ s> <\x00\x00\x01\x00\x02\x012
+ s> \xa2Eerror\xa1GmessageX\x1dpath argument must be definedFstatusEerror
+ s> \r\n
+ received frame(size=60; request=1; stream=2; streamflags=stream-begin;
type=command-response; flags=eos)
+ s> 0\r\n
+ s> \r\n
+ abort: path argument must be defined!
+ [255]
+
+Unknown node is an error
+
+ $ sendhttpv2peer << EOF
+ > command filedata
+ > nodes
eval:[b'\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa']
+ > path eval:b'a'
+ > EOF
+
D4490: wireprotov2: define and implement "filedata" command
indygreg added a comment. In https://phab.mercurial-scm.org/D4490#69457, @durin42 wrote: > Oh, one more thing: I think this, as-stated, will make it at best challenging to have ACLed branches: we'd have to, on the server, walk the linknodes and see if the client was authorized to see any branch(es) that contain the node. > > That's probably okayer for us than it was for git (since we have linkrevs at least), but it's still worth considering. First, totally agree with you about wire protocol / command overhead. I plan to add bulk querying APIs later. Also, I wasn't aware of the ACL'd branches requirement. I'll have to think of ways to handle that. It may involve the server advertising the available mechanisms for requesting file data (e.g. "request by file node" and "request by changeset node") and the server not allowing certain modes. But that example would undercut raw data access. So maybe it gets advertised as a server preference. Or maybe we end up not doing the "request by file node" path in the common case and the server overhead isn't a problem in practice. I'll keep this use case in mind when writing future commits. REPOSITORY rHG Mercurial REVISION DETAIL https://phab.mercurial-scm.org/D4490 To: indygreg, #hg-reviewers, durin42 Cc: durin42, mercurial-devel ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
D4490: wireprotov2: define and implement "filedata" command
durin42 added a comment. Oh, one more thing: I think this, as-stated, will make it at best challenging to have ACLed branches: we'd have to, on the server, walk the linknodes and see if the client was authorized to see any branch(es) that contain the node. That's probably okayer for us than it was for git (since we have linkrevs at least), but it's still worth considering. REPOSITORY rHG Mercurial REVISION DETAIL https://phab.mercurial-scm.org/D4490 To: indygreg, #hg-reviewers, durin42 Cc: durin42, mercurial-devel ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
D4490: wireprotov2: define and implement "filedata" command
durin42 accepted this revision. durin42 added a comment. This revision is now accepted and ready to land. In general I'm happy with where this is going, but I'm extremely uncomfortable with the manifest and filelog commands requiring so much client-server transfer to get everything, especially the rpc-per-file nature of this. It's elegant in terms of how easy it'll make remotefilelog et al, but it makes me worried about performance in the small-repo case where today we can just issue one big hoss RPC for everything and we get it all in one go. REPOSITORY rHG Mercurial REVISION DETAIL https://phab.mercurial-scm.org/D4490 To: indygreg, #hg-reviewers, durin42 Cc: durin42, mercurial-devel ___ Mercurial-devel mailing list Mercurial-devel@mercurial-scm.org https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
D4490: wireprotov2: define and implement "filedata" command
indygreg created this revision.
Herald added a subscriber: mercurial-devel.
Herald added a reviewer: hg-reviewers.
REVISION SUMMARY
Continuing our trend of implementing *data commands for retrieving
information about specific repository data primitives, this commit
implements a command for retrieving data about an individual tracked
file.
The command is very similar to "manifestdata." The only significant
difference is that we have a standalone function for obtaining
storage for a tracked file. This is to provide a monkeypatch point
for extensions to implement path-based access control.
With this API available, wire protocol version 2 now exposes all
data primitives necessary to implement a full clone. Of course,
since "filedata" can only resolve data for a single path at a time,
clients would need to issue N commands to perform a full clone. On
the Firefox repository, this would be ~461k commands. We'll likely
need to implement a file data retrieval command that supports
multiple paths. But that can be implemented later.
REPOSITORY
rHG Mercurial
REVISION DETAIL
https://phab.mercurial-scm.org/D4490
AFFECTED FILES
mercurial/help/internals/wireprotocolv2.txt
mercurial/wireprotov2server.py
tests/test-http-protocol.t
tests/test-wireproto-command-capabilities.t
tests/test-wireproto-command-filedata.t
CHANGE DETAILS
diff --git a/tests/test-wireproto-command-filedata.t
b/tests/test-wireproto-command-filedata.t
new file mode 100644
--- /dev/null
+++ b/tests/test-wireproto-command-filedata.t
@@ -0,0 +1,497 @@
+ $ . $TESTDIR/wireprotohelpers.sh
+
+ $ hg init server
+ $ enablehttpv2 server
+ $ cd server
+ $ echo a0 > a
+ $ echo b0 > b
+ $ mkdir -p dir0/child0 dir0/child1 dir1
+ $ echo c0 > dir0/c
+ $ echo d0 > dir0/d
+ $ echo e0 > dir0/child0/e
+ $ echo f0 > dir0/child1/f
+ $ hg -q commit -A -m 'commit 0'
+
+ $ echo a1 > a
+ $ echo d1 > dir0/d
+ $ hg commit -m 'commit 1'
+ $ echo f0 > dir0/child1/f
+ $ hg commit -m 'commit 2'
+ nothing changed
+ [1]
+
+ $ hg -q up -r 0
+ $ echo a2 > a
+ $ hg commit -m 'commit 3'
+ created new head
+
+ $ hg log -G -T '{rev}:{node} {desc}\n'
+ @ 2:c8757a2ffe552850d1e0dfe60d295ebf64c196d9 commit 3
+ |
+ | o 1:650165e803375748a94df471e5b58d85763e0b29 commit 1
+ |/
+ o 0:6d85ca1270b377d320098556ba5bfad34a9ee12d commit 0
+
+
+ $ hg --debug debugindex a
+ rev linkrev nodeid p1
p2
+ 0 0 2b4eb07319bfa077a40a2f04913659aef0da42da
+ 1 1 9a38122997b3ac97be2a9aa2e556838341fdf2cc
2b4eb07319bfa077a40a2f04913659aef0da42da
+ 2 2 0879345e39377229634b420c639454156726c6b6
2b4eb07319bfa077a40a2f04913659aef0da42da
+
+ $ hg --debug debugindex dir0/child0/e
+ rev linkrev nodeid p1
p2
+ 0 0 bbba6c06b30f443d34ff841bc985c4d0827c6be4
+
+ $ hg serve -p $HGPORT -d --pid-file hg.pid -E error.log
+ $ cat hg.pid > $DAEMON_PIDS
+
+Missing arguments is an error
+
+ $ sendhttpv2peer << EOF
+ > command filedata
+ > EOF
+ creating http peer for wire protocol version 2
+ sending filedata command
+ s> POST /api/exp-http-v2-0001/ro/filedata HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> accept: application/mercurial-exp-framing-0005\r\n
+ s> content-type: application/mercurial-exp-framing-0005\r\n
+ s> content-length: 23\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s> user-agent: Mercurial debugwireproto\r\n
+ s> \r\n
+ s> \x0f\x00\x00\x01\x00\x01\x01\x11\xa1DnameHfiledata
+ s> makefile('rb', None)
+ s> HTTP/1.1 200 OK\r\n
+ s> Server: testing stub value\r\n
+ s> Date: $HTTP_DATE$\r\n
+ s> Content-Type: application/mercurial-exp-framing-0005\r\n
+ s> Transfer-Encoding: chunked\r\n
+ s> \r\n
+ s> 45\r\n
+ s> =\x00\x00\x01\x00\x02\x012
+ s> \xa2Eerror\xa1GmessageX\x1enodes argument must be definedFstatusEerror
+ s> \r\n
+ received frame(size=61; request=1; stream=2; streamflags=stream-begin;
type=command-response; flags=eos)
+ s> 0\r\n
+ s> \r\n
+ abort: nodes argument must be defined!
+ [255]
+
+ $ sendhttpv2peer << EOF
+ > command filedata
+ > nodes eval:[]
+ > EOF
+ creating http peer for wire protocol version 2
+ sending filedata command
+ s> POST /api/exp-http-v2-0001/ro/filedata HTTP/1.1\r\n
+ s> Accept-Encoding: identity\r\n
+ s> accept: application/mercurial-exp-framing-0005\r\n
+ s> content-type: application/mercurial-exp-framing-0005\r\n
+ s> content-length: 36\r\n
+ s> host: $LOCALIP:$HGPORT\r\n (glob)
+ s>
