This is an automated email from the ASF dual-hosted git repository.
toulmean pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-tuweni.git
The following commit(s) were added to refs/heads/main by this push:
new 90ea925 Configure TODOs - move to config
new 8c8c4c6 Merge pull request #294 from atoulme/configure_todos
90ea925 is described below
commit 90ea92501f341a17f546238e6b447f2254b06dee
Author: Antoine Toulme <[email protected]>
AuthorDate: Mon Jun 28 22:44:36 2021 +0200
Configure TODOs - move to config
---
.../kotlin/org/apache/tuweni/eth/crawler/CrawlerApp.kt | 8 ++++----
.../org/apache/tuweni/eth/crawler/CrawlerConfig.kt | 17 +++++++++++++++--
.../org/apache/tuweni/eth/crawler/CrawlerRESTService.kt | 4 ++--
.../tuweni/eth/crawler/RelationalPeerRepository.kt | 6 ++++--
4 files changed, 25 insertions(+), 10 deletions(-)
diff --git
a/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/CrawlerApp.kt
b/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/CrawlerApp.kt
index 444f8ea..c6fb3fb 100644
--- a/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/CrawlerApp.kt
+++ b/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/CrawlerApp.kt
@@ -93,7 +93,7 @@ class CrawlerApplication(
fun run(vertx: Vertx, config: CrawlerConfig) {
val dbConfig = HikariConfig()
- dbConfig.maximumPoolSize = 25 // TODO make this a config setting.
+ dbConfig.maximumPoolSize = config.jdbcConnections()
dbConfig.jdbcUrl = config.jdbcUrl()
val ds = HikariDataSource(dbConfig)
val flyway = Flyway.configure()
@@ -101,7 +101,7 @@ class CrawlerApplication(
.load()
flyway.migrate()
- val repo = RelationalPeerRepository(ds)
+ val repo = RelationalPeerRepository(ds, config.peerCacheExpiration(),
config.clientIdsInterval(), coroutineContext)
logger.info("Initial bootnodes: ${config.bootNodes()}")
val scraper = Scraper(
@@ -156,7 +156,7 @@ class CrawlerApplication(
}
wireConnectionsRepository.addConnectionListener {
launch {
- delay(10 * 1000) // TODO this delay can be made configurable.
+ delay(config.rlpxDisconnectionDelay())
it.disconnect(DisconnectReason.CLIENT_QUITTING)
}
}
@@ -178,7 +178,7 @@ class CrawlerApplication(
}
}
val restService =
- CrawlerRESTService(port = config.restPort(), networkInterface =
config.restNetworkInterface(), repository = repo)
+ CrawlerRESTService(port = config.restPort(), networkInterface =
config.restNetworkInterface(), repository = repo, maxRequestsPerSec =
config.maxRequestsPerSec())
val refreshLoop = AtomicBoolean(true)
val ethstatsDataRepository = EthstatsDataRepository(ds)
val ethstatsServer = EthStatsServer(
diff --git
a/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/CrawlerConfig.kt
b/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/CrawlerConfig.kt
index ae21c87..1066272 100644
--- a/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/CrawlerConfig.kt
+++ b/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/CrawlerConfig.kt
@@ -55,17 +55,23 @@ class CrawlerConfig(val filePath: Path) {
.addString("rlpxNetworkInterface", "127.0.0.1", "RLPx network
interface", null)
.addListOfString("bootnodes", mainnetEthereumBootnodes, "Bootnodes to
discover other peers from", null)
.addString("discoveryDNS", mainnetDiscoveryDNS, "DNS discovery crawler",
null)
- .addLong("discoveryDNSPollingPeriod", 60 * 1000, "DNS Discovery Polling
Period in milliseconds", null)
+ .addLong("discoveryDNSPollingPeriod", 60 * 1000L, "DNS Discovery Polling
Period in milliseconds", null)
.addString(
"jdbcUrl", System.getProperty("DATABASE_URL",
System.getenv("DATABASE_URL")),
"JDBC URL of the form jdbc:posgresql://localhost:5432",
PropertyValidator.isPresent()
- ).addString("network", "mainnet", "Network to use instead of providing a
genesis file.", null)
+ )
+ .addInteger("jdbcConnections", 25, "Number of JDBC connections for the
connections pool", null)
+ .addString("network", "mainnet", "Network to use instead of providing a
genesis file.", null)
.addString("genesisFile", "", "Genesis file to use in hello", null)
.addInteger("restPort", 1337, "REST port", null)
.addString("restNetworkInterface", "0.0.0.0", "REST network interface",
null)
.addString("ethstatsNetworkInterface", "0.0.0.0", "Ethstats network
interface", null)
.addInteger("ethstatsPort", 1338, "Ethstats port", null)
.addString("ethstatsSecret", "changeme", "Ethstats shared secret", null)
+ .addLong("peerCacheExpiration", 5 * 60 * 1000L, "Peer data cache
expiration", null)
+ .addLong("clientIdsInterval", 24 * 60 * 60 * 1000 * 2L, "Client IDs
Interval - number of milliseconds to go back in time", null)
+ .addLong("rlpxDisconnectionDelay", 10 * 1000L, "RLPx connections
disconnection delay", null)
+ .addInteger("maxRequestsPerSec", 30, "Number of requests per second over
HTTP", null)
.toSchema()
}
@@ -91,4 +97,11 @@ class CrawlerConfig(val filePath: Path) {
fun ethstatsPort() = config.getInteger("ethstatsPort")
fun ethstatsNetworkInterface() = config.getString("ethstatsNetworkInterface")
fun ethstatsSecret() = config.getString("ethstatsSecret")
+
+ fun peerCacheExpiration() = config.getLong("peerCacheExpiration")
+ fun clientIdsInterval() = config.getLong("clientIdsInterval")
+
+ fun jdbcConnections() = config.getInteger("jdbcConnections")
+ fun rlpxDisconnectionDelay() = config.getLong("rlpxDisconnectionsDelay")
+ fun maxRequestsPerSec() = config.getInteger("maxRequestsPerSec")
}
diff --git
a/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/CrawlerRESTService.kt
b/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/CrawlerRESTService.kt
index 6704f57..48ba99e 100644
---
a/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/CrawlerRESTService.kt
+++
b/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/CrawlerRESTService.kt
@@ -38,6 +38,7 @@ class CrawlerRESTService(
val port: Int = 0,
val networkInterface: String = "127.0.0.1",
val path: String = "/",
+ val maxRequestsPerSec: Int = 30,
val repository: RelationalPeerRepository,
override val coroutineContext: CoroutineContext = Dispatchers.Default,
) : CoroutineScope {
@@ -83,8 +84,7 @@ class CrawlerRESTService(
ctx.addServlet(swagger, "/swagger-ui/*")
val filter = DoSFilter()
- // TODO make it a config setting. Change for REST vs UI.
- filter.maxRequestsPerSec = 30
+ filter.maxRequestsPerSec = maxRequestsPerSec
ctx.addFilter(FilterHolder(filter), "/*",
EnumSet.of(DispatcherType.REQUEST))
newServer.stopAtShutdown = true
diff --git
a/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/RelationalPeerRepository.kt
b/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/RelationalPeerRepository.kt
index b2393e8..03c4dc2 100644
---
a/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/RelationalPeerRepository.kt
+++
b/eth-crawler/src/main/kotlin/org/apache/tuweni/eth/crawler/RelationalPeerRepository.kt
@@ -43,6 +43,8 @@ import kotlin.coroutines.CoroutineContext
open class RelationalPeerRepository(
private val dataSource: DataSource,
+ private val expiration: Long = 5 * 60 * 1000,
+ private val clientIdsInterval: Long = 24 * 60 * 60 * 1000 * 2,
override val coroutineContext: CoroutineContext = Dispatchers.Default,
) : CoroutineScope, PeerRepository {
@@ -73,7 +75,7 @@ open class RelationalPeerRepository(
}
fun get(nodeId: SECP256K1.PublicKey, endpoint: Endpoint): Peer {
- val id = peerCache.computeIfAbsent(nodeId, 5 * 60 * 60) { // 5 minutes,
TODO configure.
+ val id = peerCache.computeIfAbsent(nodeId, expiration) {
dataSource.connection.use { conn ->
logger.trace("Get peer with $nodeId")
val stmt = conn.prepareStatement("select id,publickey from identity
where publickey=?")
@@ -261,7 +263,7 @@ open class RelationalPeerRepository(
"select clients.clientId, count(clients.clientId) from (select
nodeinfo.clientId, nodeInfo.createdAt from nodeinfo inner join (select
identity, max(createdAt) as maxCreatedAt from nodeinfo group by identity)
maxSeen on nodeinfo.identity = maxSeen.identity and nodeinfo.createdAt =
maxSeen.maxCreatedAt) as clients where clients.createdAt > ? group by
clients.clientId"
)
stmt.use {
- it.setTimestamp(1, Timestamp(System.currentTimeMillis() - 24 * 60 * 60
* 1000 * 2)) // TODO configure right now, it's clients from up to 2 days ago.
+ it.setTimestamp(1, Timestamp(System.currentTimeMillis() -
clientIdsInterval))
// map results.
val rs = stmt.executeQuery()
val result = mutableListOf<ClientInfo>()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
