This is an automated email from the ASF dual-hosted git repository.
toulmean pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-tuweni.git
The following commit(s) were added to refs/heads/main by this push:
new b6ec34c add london stats
new 198bce0 Merge pull request #290 from atoulme/london_stats
b6ec34c is described below
commit b6ec34c8d5fb7b5c73ba6eb850217abcfd8ab45b
Author: Antoine Toulme <[email protected]>
AuthorDate: Sat Jun 26 19:10:31 2021 +0200
add london stats
---
.../org/apache/tuweni/devp2p/DiscoveryService.kt | 5 +++
.../tuweni/eth/crawler/RelationalPeerRepository.kt | 44 ++++++++++++++++++----
.../rest/{ClientsService.kt => ClientVersion.kt} | 31 ++++-----------
.../tuweni/eth/crawler/rest/ClientsService.kt | 11 ++++++
4 files changed, 60 insertions(+), 31 deletions(-)
diff --git
a/devp2p/src/main/kotlin/org/apache/tuweni/devp2p/DiscoveryService.kt
b/devp2p/src/main/kotlin/org/apache/tuweni/devp2p/DiscoveryService.kt
index 6638d6a..a79bc3e 100644
--- a/devp2p/src/main/kotlin/org/apache/tuweni/devp2p/DiscoveryService.kt
+++ b/devp2p/src/main/kotlin/org/apache/tuweni/devp2p/DiscoveryService.kt
@@ -889,6 +889,11 @@ internal class CoroutineDiscoveryService constructor(
logger.warn("{}: findNode state for {} has been replaced")
close()
}
+ } catch (e: TimeoutCancellationException) {
+ logger.debug(
+ "$serviceDescriptor: Timeout while sending FindNode requests for
peer ${peer.nodeId}",
+ e
+ )
} catch (e: Exception) {
logger.error(
"$serviceDescriptor: Error while sending FindNode requests for peer
${peer.nodeId}",
diff --git
a/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/RelationalPeerRepository.kt
b/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/RelationalPeerRepository.kt
index e53ba61..b2393e8 100644
---
a/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/RelationalPeerRepository.kt
+++
b/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/RelationalPeerRepository.kt
@@ -31,6 +31,7 @@ import org.apache.tuweni.devp2p.Peer
import org.apache.tuweni.devp2p.PeerRepository
import org.apache.tuweni.devp2p.eth.Status
import org.apache.tuweni.devp2p.parseEnodeUri
+import org.apache.tuweni.eth.crawler.rest.ClientVersion
import org.apache.tuweni.rlpx.wire.WireConnection
import org.slf4j.LoggerFactory
import java.net.URI
@@ -47,8 +48,19 @@ open class RelationalPeerRepository(
companion object {
internal val logger =
LoggerFactory.getLogger(RelationalPeerRepository::class.java)
- }
+ // TODO make this configurable
+ val londonClientVersions = mutableMapOf(
+ Pair("geth", "v1.10.4"),
+ Pair("nethermind", "v1.10.73"),
+ Pair("turbogeth", "v2021.06.04-alpha"),
+ Pair("turbo-geth", "v2021.06.04-alpha"),
+ Pair("erigon", "v2021.06.04-alpha"),
+ Pair("besu", "v21.7.0-RC1"),
+ Pair("openethereum", "v3.3.0-rc2"),
+ Pair("ethereumjs", "v5.4.1")
+ )
+ }
private val listeners = mutableListOf<(Peer) -> Unit>()
private val peerCache = ExpiringMap<SECP256K1.PublicKey, String>()
@@ -108,15 +120,15 @@ open class RelationalPeerRepository(
"insert into nodeInfo(id, createdAt, host, port, publickey,
p2pVersion, clientId, capabilities, genesisHash, bestHash, totalDifficulty,
identity, disconnectReason) values(?,?,?,?,?,?,?,?,?,?,?,?,?)"
)
stmt.use {
- val peerHello = wireConnection.peerHello!!
+ val peerHello = wireConnection.peerHello
it.setString(1, UUID.randomUUID().toString())
it.setTimestamp(2, Timestamp(System.currentTimeMillis()))
it.setString(3, wireConnection.peerHost())
it.setInt(4, wireConnection.peerPort())
it.setBytes(5, wireConnection.peerPublicKey().bytesArray())
- it.setInt(6, peerHello.p2pVersion())
- it.setString(7, peerHello.clientId())
- it.setString(8, peerHello.capabilities().map { it.name() + "/" +
it.version() }.joinToString(","))
+ it.setInt(6, peerHello?.p2pVersion() ?: 0)
+ it.setString(7, peerHello?.clientId() ?: "")
+ it.setString(8, peerHello?.capabilities()?.joinToString(",") {
it.name() + "/" + it.version() } ?: "")
it.setString(9, status?.genesisHash?.toHexString())
it.setString(10, status?.bestHash?.toHexString())
it.setString(11, status?.totalDifficulty?.toHexString())
@@ -208,13 +220,27 @@ open class RelationalPeerRepository(
}
private var clientIds: List<ClientInfo>? = null
+ private var londonStats: ClientReadyStats? = null
private val started = AtomicBoolean(false)
fun start() {
launch {
started.set(true)
while (started.get()) {
- clientIds = getClientIdsInternal()
+ val newClientIds = getClientIdsInternal()
+ clientIds = newClientIds
+ var londonReady = 0
+ var total = 0
+ newClientIds.forEach { newClientCount ->
+ total += newClientCount.count
+ val clientVersion = ClientVersion(newClientCount.clientId)
+ londonClientVersions[clientVersion.name().toLowerCase()]?.let {
londonVersion ->
+ if (clientVersion >= londonVersion) {
+ londonReady += newClientCount.count
+ }
+ }
+ }
+ londonStats = ClientReadyStats(total, londonReady)
delay(30 * 1000)
}
}
@@ -224,15 +250,18 @@ open class RelationalPeerRepository(
started.set(false)
}
+ internal fun getLondonStats() = londonStats
+
internal fun getClientIds(): List<ClientInfo> = clientIds ?: listOf()
internal fun getClientIdsInternal(): List<ClientInfo> {
dataSource.connection.use { conn ->
val stmt =
conn.prepareStatement(
- "select clients.clientId, count(clients.clientId) from (select
nodeinfo.clientId from nodeinfo inner join (select identity, max(createdAt) as
maxCreatedAt from nodeinfo group by identity) maxSeen on nodeinfo.identity =
maxSeen.identity and nodeinfo.createdAt = maxSeen.maxCreatedAt) as clients
group by clients.clientId"
+ "select clients.clientId, count(clients.clientId) from (select
nodeinfo.clientId, nodeInfo.createdAt from nodeinfo inner join (select
identity, max(createdAt) as maxCreatedAt from nodeinfo group by identity)
maxSeen on nodeinfo.identity = maxSeen.identity and nodeinfo.createdAt =
maxSeen.maxCreatedAt) as clients where clients.createdAt > ? group by
clients.clientId"
)
stmt.use {
+ it.setTimestamp(1, Timestamp(System.currentTimeMillis() - 24 * 60 * 60
* 1000 * 2)) // TODO configure right now, it's clients from up to 2 days ago.
// map results.
val rs = stmt.executeQuery()
val result = mutableListOf<ClientInfo>()
@@ -246,6 +275,7 @@ open class RelationalPeerRepository(
}
}
}
+internal data class ClientReadyStats(val total: Int, val ready: Int)
internal data class ClientInfo(val clientId: String, val count: Int)
internal data class PeerConnectionInfo(val nodeId: SECP256K1.PublicKey, val
host: String, val port: Int)
internal data class PeerConnectionInfoDetails(val createdAt: Long, val nodeId:
SECP256K1.PublicKey, val p2pVersion: Int, val clientId: String, val
capabilities: String, val genesisHash: String, val bestHash: String, val
totalDifficulty: String)
diff --git
a/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/rest/ClientsService.kt
b/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/rest/ClientVersion.kt
similarity index 52%
copy from
eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/rest/ClientsService.kt
copy to
eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/rest/ClientVersion.kt
index f692018..2e73f3e 100644
---
a/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/rest/ClientsService.kt
+++
b/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/rest/ClientVersion.kt
@@ -16,34 +16,17 @@
*/
package org.apache.tuweni.eth.crawler.rest
-import com.fasterxml.jackson.databind.ObjectMapper
-import org.apache.tuweni.eth.EthJsonModule
-import org.apache.tuweni.eth.crawler.RelationalPeerRepository
-import javax.servlet.ServletContext
-import javax.ws.rs.GET
-import javax.ws.rs.Path
-import javax.ws.rs.Produces
-import javax.ws.rs.core.MediaType
+data class ClientVersion(val clientId: String) : Comparable<String> {
-@Path("clients")
-class ClientsService {
+ val segments = clientId.split("/")
- companion object {
- val mapper = ObjectMapper()
+ fun name() = segments[0]
- init {
- mapper.registerModule(EthJsonModule())
- }
+ fun version() = segments.firstOrNull() {
+ it.startsWith("v")
}
- @javax.ws.rs.core.Context
- var context: ServletContext? = null
- @GET
- @Produces(MediaType.APPLICATION_JSON)
- fun getClientIds(): String {
- val repo = context!!.getAttribute("repo") as RelationalPeerRepository
- val peers = repo.getClientIds()
- val result = mapper.writeValueAsString(peers)
- return result
+ override fun compareTo(other: String): Int {
+ return version()?.compareTo(other) ?: 1
}
}
diff --git
a/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/rest/ClientsService.kt
b/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/rest/ClientsService.kt
index f692018..afcaf9b 100644
---
a/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/rest/ClientsService.kt
+++
b/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/rest/ClientsService.kt
@@ -40,10 +40,21 @@ class ClientsService {
@GET
@Produces(MediaType.APPLICATION_JSON)
+ @Path("all")
fun getClientIds(): String {
val repo = context!!.getAttribute("repo") as RelationalPeerRepository
val peers = repo.getClientIds()
val result = mapper.writeValueAsString(peers)
return result
}
+
+ @GET
+ @Produces(MediaType.APPLICATION_JSON)
+ @Path("london/stats")
+ fun getClientStats(): String {
+ val repo = context!!.getAttribute("repo") as RelationalPeerRepository
+ val peers = repo.getLondonStats()
+ val result = mapper.writeValueAsString(peers)
+ return result
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
