This is an automated email from the ASF dual-hosted git repository. dpavlov pushed a commit to branch ignite-10942 in repository https://gitbox.apache.org/repos/asf/ignite-teamcity-bot.git
commit 72d0813bacb1d7eec3920791982a1bddf3ebb195 Author: Dmitriy Pavlov <[email protected]> AuthorDate: Thu Jan 17 21:26:51 2019 +0300 IGNITE-10942:: Optimize background JIRA tickets sync --- .../ci/github/ignited/GitHubConnIgnitedImpl.java | 11 ++--- .../ignite/ci/jira/ignited/JiraTicketDao.java | 25 ++++++++--- .../ignite/ci/jira/ignited/JiraTicketSync.java | 51 +++++++++++++++------- .../ignite/ci/jira/ignited/TicketCompacted.java | 4 +- .../ignite/ci/jira/pure/IJiraIntegration.java | 3 ++ .../java/org/apache/ignite/ci/jira/pure/Jira.java | 9 ++-- .../ci/teamcity/ignited/TeamcityIgnitedImpl.java | 2 +- 7 files changed, 68 insertions(+), 37 deletions(-) diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java index 2838936..3cbe059 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/github/ignited/GitHubConnIgnitedImpl.java @@ -110,14 +110,9 @@ class GitHubConnIgnitedImpl implements IGitHubConnIgnited { runActualizePrs(srvId, false); // schedule full resync later - scheduler.invokeLater(this::sheduleResync, 20, TimeUnit.SECONDS); - } - - /** - * - */ - private void sheduleResync() { - scheduler.sheduleNamed(taskName("fullReindex"), this::fullReindex, 60, TimeUnit.MINUTES); + scheduler.invokeLater( + () -> scheduler.sheduleNamed(taskName("fullReindex"), this::fullReindex, 2, TimeUnit.HOURS), + 1, TimeUnit.MINUTES); } /** diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketDao.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketDao.java index 6502488..30def78 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketDao.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketDao.java @@ -19,6 +19,7 @@ package org.apache.ignite.ci.jira.ignited; import com.google.common.base.Preconditions; import java.util.Collection; +import java.util.Map; import java.util.HashMap; import java.util.HashSet; import java.util.Set; @@ -89,19 +90,19 @@ public class JiraTicketDao { /** * Save small part of loaded mutes. - * * @param srvIdMaskHigh Server id mask high. * @param chunk Chunk. * @param ticketPrefix Ticket name template. + * @return number of tickets totally saved. */ @AutoProfiling - public void saveChunk(int srvIdMaskHigh, Collection<Ticket> chunk, String ticketPrefix) { + public int saveChunk(int srvIdMaskHigh, Collection<Ticket> chunk, String ticketPrefix) { Preconditions.checkNotNull(jiraCache, "init() was not called"); if (F.isEmpty(chunk)) - return; + return 0; - HashMap<Long, TicketCompacted> compactedTickets = new HashMap<>(U.capacity(chunk.size())); + Map<Long, TicketCompacted> compactedTickets = new HashMap<>(U.capacity(chunk.size())); for (Ticket ticket : chunk) { long key = ticketToCacheKey(srvIdMaskHigh, ticket.igniteId(ticketPrefix)); @@ -110,6 +111,20 @@ public class JiraTicketDao { compactedTickets.put(key, val); } - jiraCache.putAll(compactedTickets); + Map<Long, TicketCompacted> dbVal = jiraCache.getAll(compactedTickets.keySet()); + + Map<Long, TicketCompacted> ticketsToUpdate = new HashMap<>(U.capacity(chunk.size())); + + compactedTickets.forEach((k, v) -> { + TicketCompacted existing = dbVal.get(k); + + if (existing == null || !v.equals(existing)) + ticketsToUpdate.put(k, v); + }); + + if(!ticketsToUpdate.isEmpty()) + jiraCache.putAll(ticketsToUpdate); + + return ticketsToUpdate.size(); } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketSync.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketSync.java index 178ced6..598f074 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketSync.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/JiraTicketSync.java @@ -49,8 +49,23 @@ public class JiraTicketSync { * @param srvId Server ID */ public void ensureActualizeJiraTickets(String srvId) { - scheduler.sheduleNamed(taskName("actualizeJiraTickets", srvId), - () -> actualizeJiraTickets(srvId), 15, TimeUnit.MINUTES); + scheduler.sheduleNamed(taskName("incrementalSync", srvId), + () -> incrementalUpdate(srvId), 15, TimeUnit.MINUTES); + } + + /** + * @param srvId Server id. + */ + public String incrementalUpdate(String srvId) { + String res = actualizeJiraTickets(srvId, false); + + scheduler.invokeLater(() -> { + scheduler.sheduleNamed(taskName("fullResync", srvId), + () -> actualizeJiraTickets(srvId, true), 2, TimeUnit.HOURS); + }, + 5, TimeUnit.MINUTES); + + return res; } /** @@ -64,9 +79,10 @@ public class JiraTicketSync { } /** * @param srvId Server internal identification. + * @param fullResync full or incremental. */ @MonitoredTask(name = "Actualize Jira", nameExtArgsIndexes = {0}) - protected String actualizeJiraTickets(String srvId) { + protected String actualizeJiraTickets(String srvId, boolean fullResync) { int srvIdMaskHigh = ITeamcityIgnited.serverIdToInt(srvId); IJiraIntegration jira = jiraIntegrationProvider.server(srvId); @@ -81,26 +97,31 @@ public class JiraTicketSync { if (F.isEmpty(page)) return "Something went wrong - no tickets found. Check jira availability."; - jiraDao.saveChunk(srvIdMaskHigh, page, jira.ticketPrefix()); + int ticketsSaved = jiraDao.saveChunk(srvIdMaskHigh, page, jira.ticketPrefix()); + + int ticketsProcessed = page.size(); - int ticketsSaved = page.size(); + if (ticketsSaved != 0 || fullResync) { + while (tickets.nextStart() > 0) { + url = baseUrl + "&startAt=" + tickets.nextStart(); - while (tickets.nextStart() > 0) { - url = baseUrl + "&startAt=" + tickets.nextStart(); + tickets = jira.getTicketsPage(srvId, url); - tickets = jira.getTicketsPage(srvId, url); + page = tickets.issuesNotNull(); - page = tickets.issuesNotNull(); + if (F.isEmpty(page)) + break; - if (F.isEmpty(page)) - break; + int savedNow = jiraDao.saveChunk(srvIdMaskHigh, page, jira.ticketPrefix()); - //todo find not updated chunk and exit - jiraDao.saveChunk(srvIdMaskHigh, page, jira.ticketPrefix()); + ticketsSaved += savedNow; + ticketsProcessed += page.size(); - ticketsSaved += page.size(); + if (savedNow == 0 && !fullResync) + break; // find not updated chunk and exit + } } - return "Jira tickets saved " + ticketsSaved + " for " + srvId; + return "Jira tickets saved " + ticketsSaved + " from" + ticketsProcessed + " for " + srvId; } } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/TicketCompacted.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/TicketCompacted.java index d1a22f9..4eb942e 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/TicketCompacted.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/ignited/TicketCompacted.java @@ -30,10 +30,10 @@ public class TicketCompacted { /** Id. */ public long id; - /** Ticket full name like "IGNITE-123". */ + /** Ticket number, integer value like 123 from name like "IGNITE-123". */ public int igniteId; - /** Fields. */ + /** Id of string: Fields/status/name, value compacted. */ public int status; /** diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/IJiraIntegration.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/IJiraIntegration.java index 6ff8fea..ec4c782 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/IJiraIntegration.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/IJiraIntegration.java @@ -81,5 +81,8 @@ public interface IJiraIntegration { String getJiraApiUrl(); + /** + * @return {@code True} if JIRA authorization token is available. + */ boolean isJiraTokenAvailable(); } diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/Jira.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/Jira.java index e5a006e..3fd177e 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/Jira.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/jira/pure/Jira.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; import static com.google.common.base.Strings.isNullOrEmpty; /** - * + * Implementation of pure JIRA interaction. */ class Jira implements IJiraIntegration { /** Logger. */ @@ -51,6 +51,8 @@ class Jira implements IJiraIntegration { /** URL for JIRA integration. */ private String jiraApiUrl; + + /** Server id. */ private String srvId; /** {@inheritDoc} */ @@ -70,11 +72,6 @@ class Jira implements IJiraIntegration { jiraApiUrl = props.getProperty(HelperConfig.JIRA_API_URL); } - /** - * @return {@code True} if JIRA authorization token is available. - */ - // boolean isJiraTokenAvailable(); - /** {@inheritDoc} */ @Override public String jiraUrl() { return jiraUrl; diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java index 3938c27..073f1a7 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/teamcity/ignited/TeamcityIgnitedImpl.java @@ -597,7 +597,7 @@ public class TeamcityIgnitedImpl implements ITeamcityIgnited { * */ private void sheduleResyncBuildRefs() { - scheduler.sheduleNamed(taskName("fullReindex"), this::fullReindex, 120, TimeUnit.MINUTES); + scheduler.sheduleNamed(taskName("fullReindex"), this::fullReindex, 2, TimeUnit.HOURS); } /**
