提交 72a7529a 编写于 作者: O Oliver Gondža 提交者: Oleg Nenashev

[FIX JENKINS-30909] Make sure queue is persisted reliably (#3244)

* [FIX JENKINS-30909] Make sure queue is persisted reliably

* Fix tests on Windows

* Ensure Saver is thread-safe
上级 72d80b49
......@@ -24,10 +24,12 @@
*/
package hudson.model;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.infradna.tool.bridge_method_injector.WithBridgeMethods;
import hudson.BulkChange;
import hudson.Extension;
import hudson.ExtensionList;
import hudson.ExtensionPoint;
import hudson.Util;
......@@ -64,7 +66,10 @@ import hudson.model.queue.WorkUnitContext;
import hudson.security.ACL;
import hudson.security.AccessControlled;
import java.nio.file.Files;
import hudson.util.Futures;
import jenkins.security.QueueItemAuthenticatorProvider;
import jenkins.util.SystemProperties;
import jenkins.util.Timer;
import hudson.triggers.SafeTimerTask;
import java.util.concurrent.TimeUnit;
......@@ -93,7 +98,6 @@ import java.util.NoSuchElementException;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Condition;
......@@ -102,6 +106,7 @@ import java.util.logging.Level;
import java.util.logging.Logger;
import javax.annotation.Nonnull;
import javax.annotation.concurrent.GuardedBy;
import javax.servlet.ServletException;
import jenkins.model.Jenkins;
......@@ -3009,4 +3014,75 @@ public class Queue extends ResourceController implements Saveable {
public static void init(Jenkins h) {
h.getQueue().load();
}
/**
* Schedule <tt>Queue.save()</tt> call for near future once items change. Ignore all changes until the time the save
* takes place.
*
* Once queue is restored after a crash, items stages might not be accurate until the next #maintain() - this is not
* a problem as the items will be reshuffled first and then scheduled during the next maintainance cycle.
*
* Implementation note: Queue.load() calls QueueListener hooks for every item deserialized that can hammer the persistance
* on load. The problem is avoided by delaying the actual save for the time long enough for queue to load so the save
* operations will collapse into one. Also, items are persisted as buildable or blocked in vast majority of cases and
* those stages does not trigger the save here.
*/
@Extension
@Restricted(NoExternalUse.class)
public static final class Saver extends QueueListener implements Runnable {
/**
* All negative values will disable periodic saving.
*/
@VisibleForTesting
/*package*/ static /*final*/ int DELAY_SECONDS = SystemProperties.getInteger("hudson.model.Queue.Saver.DELAY_SECONDS", 60);
private final Object lock = new Object();
@GuardedBy("lock")
private Future<?> nextSave;
@Override
public void onEnterWaiting(WaitingItem wi) {
push();
}
@Override
public void onLeft(Queue.LeftItem li) {
push();
}
private void push() {
if (DELAY_SECONDS < 0) return;
synchronized (lock) {
// Can be done or canceled in case of a bug or external intervention - do not allow it to hang there forever
if (nextSave != null && !(nextSave.isDone() || nextSave.isCancelled())) return;
nextSave = Timer.get().schedule(this, DELAY_SECONDS, TimeUnit.SECONDS);
}
}
@Override
public void run() {
try {
Jenkins j = Jenkins.getInstanceOrNull();
if (j != null) {
j.getQueue().save();
}
} finally {
synchronized (lock) {
nextSave = null;
}
}
}
@VisibleForTesting @Restricted(NoExternalUse.class)
/*package*/ @Nonnull Future<?> getNextSave() {
synchronized (lock) {
return nextSave == null
? Futures.precomputed(null)
: nextSave
;
}
}
}
}
......@@ -53,7 +53,7 @@ THE SOFTWARE.
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jenkins-test-harness</artifactId>
<version>2.32</version>
<version>2.33</version>
<scope>test</scope>
<exclusions>
<exclusion>
......
/*
* The MIT License
*
* Copyright (c) Red Hat, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
package hudson.model;
import hudson.ExtensionList;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runners.model.Statement;
import org.jvnet.hudson.test.RestartableJenkinsRule;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class QueueRestartTest {
@Rule
public RestartableJenkinsRule j = new RestartableJenkinsRule();
@Test
public void persistQueueOnRestart() {
j.addStep(new Statement() {
@Override public void evaluate() throws Throwable {
Queue.Saver.DELAY_SECONDS = 24 * 60 * 60; // Avoid period save as we are after the explicit one
scheduleSomeBuild();
assertBuildIsScheduled();
}
});
j.addStep(new Statement() {
@Override public void evaluate() {
assertBuildIsScheduled();
}
});
}
@Test
public void persistQueueOnCrash() {
j.addStepWithDirtyShutdown(new Statement() {
@Override public void evaluate() throws Throwable {
Queue.Saver.DELAY_SECONDS = 0; // Call Queue#save() on every queue modification simulating time has passed before crash
scheduleSomeBuild();
assertBuildIsScheduled();
// Save have no delay though is not synchronous
ExtensionList.lookup(Queue.Saver.class).get(0).getNextSave().get(3, TimeUnit.SECONDS);
assertTrue("queue.xml does not exist", j.j.jenkins.getQueue().getXMLQueueFile().exists());
}
});
j.addStep(new Statement() {
@Override public void evaluate() {
assertBuildIsScheduled();
}
});
}
private void assertBuildIsScheduled() {
assertEquals(1, j.j.jenkins.getQueue().getItems().length);
}
private void scheduleSomeBuild() throws IOException {
FreeStyleProject p = j.j.createFreeStyleProject();
p.setAssignedLabel(Label.get("waitforit"));
p.scheduleBuild2(0);
}
}
......@@ -46,6 +46,7 @@ import hudson.model.Queue.BlockedItem;
import hudson.model.Queue.Executable;
import hudson.model.Queue.WaitingItem;
import hudson.model.labels.LabelExpression;
import hudson.model.listeners.SaveableListener;
import hudson.model.queue.CauseOfBlockage;
import hudson.model.queue.QueueTaskDispatcher;
import hudson.model.queue.QueueTaskFuture;
......@@ -1018,4 +1019,22 @@ public class QueueTest {
assertEquals(expected.getShortDescription(), actual.getShortDescription());
}
@Test @LocalData
public void load_queue_xml() {
Queue q = r.getInstance().getQueue();
Queue.Item[] items = q.getItems();
assertEquals(Arrays.asList(items).toString(), 11, items.length);
assertEquals("Loading the queue should not generate saves", 0, QueueSaveSniffer.count);
}
@TestExtension("load_queue_xml")
public static final class QueueSaveSniffer extends SaveableListener {
private static int count = 0;
@Override public void onChange(Saveable o, XmlFile file) {
if (o instanceof Queue) {
count++;
}
}
}
}
<?xml version='1.0' encoding='UTF-8'?>
<hudson>
<disabledAdministrativeMonitors/>
<version>2.103-SNAPSHOT (private-01/18/2018 15:13 GMT-ogondza)</version>
<installState class="jenkins.install.InstallState$5">
<isSetupComplete>true</isSetupComplete>
<name>RESTART</name>
</installState>
<numExecutors>2</numExecutors>
<mode>NORMAL</mode>
<useSecurity>true</useSecurity>
<authorizationStrategy class="hudson.security.AuthorizationStrategy$Unsecured"/>
<securityRealm class="hudson.security.HudsonPrivateSecurityRealm">
<disableSignup>true</disableSignup>
<enableCaptcha>false</enableCaptcha>
</securityRealm>
<disableRememberMe>false</disableRememberMe>
<projectNamingStrategy class="jenkins.model.ProjectNamingStrategy$DefaultProjectNamingStrategy"/>
<workspaceDir>${JENKINS_HOME}/workspace/${ITEM_FULL_NAME}</workspaceDir>
<buildsDir>${ITEM_ROOTDIR}/builds</buildsDir>
<markupFormatter class="hudson.markup.EscapedMarkupFormatter"/>
<jdks/>
<viewsTabBar class="hudson.views.DefaultViewsTabBar"/>
<myViewsTabBar class="hudson.views.DefaultMyViewsTabBar"/>
<clouds/>
<scmCheckoutRetryCount>0</scmCheckoutRetryCount>
<views>
<hudson.model.AllView>
<owner class="hudson" reference="../../.."/>
<name>all</name>
<filterExecutors>false</filterExecutors>
<filterQueue>false</filterQueue>
<properties class="hudson.model.View$PropertyList"/>
</hudson.model.AllView>
</views>
<primaryView>all</primaryView>
<slaveAgentPort>-1</slaveAgentPort>
<disabledAgentProtocols>
<string>JNLP-connect</string>
<string>JNLP2-connect</string>
</disabledAgentProtocols>
<label></label>
<crumbIssuer class="hudson.security.csrf.DefaultCrumbIssuer">
<excludeClientIPFromCrumb>false</excludeClientIPFromCrumb>
</crumbIssuer>
<nodeProperties/>
<globalNodeProperties/>
</hudson>
\ No newline at end of file
<?xml version='1.0' encoding='UTF-8'?>
<project>
<actions/>
<description></description>
<keepDependencies>false</keepDependencies>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<canRoam>false</canRoam>
<assignedNode>!master</assignedNode>
<disabled>false</disabled>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<triggers/>
<concurrentBuild>false</concurrentBuild>
<builders/>
<publishers/>
<buildWrappers/>
</project>
<?xml version='1.0' encoding='UTF-8'?>
<project>
<actions/>
<description></description>
<keepDependencies>false</keepDependencies>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<canRoam>false</canRoam>
<assignedNode>!master</assignedNode>
<disabled>false</disabled>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<triggers/>
<concurrentBuild>false</concurrentBuild>
<builders/>
<publishers/>
<buildWrappers/>
</project>
<?xml version='1.0' encoding='UTF-8'?>
<project>
<actions/>
<description></description>
<keepDependencies>false</keepDependencies>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<canRoam>false</canRoam>
<assignedNode>!master</assignedNode>
<disabled>false</disabled>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<triggers/>
<concurrentBuild>false</concurrentBuild>
<builders/>
<publishers/>
<buildWrappers/>
</project>
<?xml version='1.0' encoding='UTF-8'?>
<project>
<actions/>
<description></description>
<keepDependencies>false</keepDependencies>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<canRoam>false</canRoam>
<assignedNode>!master</assignedNode>
<disabled>false</disabled>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<triggers/>
<concurrentBuild>false</concurrentBuild>
<builders/>
<publishers/>
<buildWrappers/>
</project>
<?xml version='1.0' encoding='UTF-8'?>
<project>
<actions/>
<description></description>
<keepDependencies>false</keepDependencies>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<canRoam>false</canRoam>
<assignedNode>!master</assignedNode>
<disabled>false</disabled>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<triggers/>
<concurrentBuild>false</concurrentBuild>
<builders/>
<publishers/>
<buildWrappers/>
</project>
<?xml version='1.0' encoding='UTF-8'?>
<project>
<actions/>
<description></description>
<keepDependencies>false</keepDependencies>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<canRoam>false</canRoam>
<assignedNode>!master</assignedNode>
<disabled>false</disabled>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<triggers/>
<concurrentBuild>false</concurrentBuild>
<builders/>
<publishers/>
<buildWrappers/>
</project>
<?xml version='1.0' encoding='UTF-8'?>
<project>
<actions/>
<description></description>
<keepDependencies>false</keepDependencies>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<canRoam>false</canRoam>
<assignedNode>!master</assignedNode>
<disabled>false</disabled>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<triggers/>
<concurrentBuild>false</concurrentBuild>
<builders/>
<publishers/>
<buildWrappers/>
</project>
<?xml version='1.0' encoding='UTF-8'?>
<project>
<actions/>
<description></description>
<keepDependencies>false</keepDependencies>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<canRoam>false</canRoam>
<assignedNode>!master</assignedNode>
<disabled>false</disabled>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<triggers/>
<concurrentBuild>false</concurrentBuild>
<builders/>
<publishers/>
<buildWrappers/>
</project>
<?xml version='1.0' encoding='UTF-8'?>
<project>
<actions/>
<description></description>
<keepDependencies>false</keepDependencies>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<canRoam>false</canRoam>
<assignedNode>!master</assignedNode>
<disabled>false</disabled>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<triggers/>
<concurrentBuild>false</concurrentBuild>
<builders/>
<publishers/>
<buildWrappers/>
</project>
<?xml version='1.0' encoding='UTF-8'?>
<project>
<actions/>
<description></description>
<keepDependencies>false</keepDependencies>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<canRoam>false</canRoam>
<assignedNode>!master</assignedNode>
<disabled>false</disabled>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<triggers/>
<concurrentBuild>false</concurrentBuild>
<builders/>
<publishers/>
<buildWrappers/>
</project>
<?xml version='1.0' encoding='UTF-8'?>
<project>
<actions/>
<description></description>
<keepDependencies>false</keepDependencies>
<properties/>
<scm class="hudson.scm.NullSCM"/>
<canRoam>false</canRoam>
<assignedNode>!master</assignedNode>
<disabled>false</disabled>
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
<blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
<triggers/>
<concurrentBuild>false</concurrentBuild>
<builders/>
<publishers/>
<buildWrappers/>
</project>
<?xml version='1.0' encoding='UTF-8'?>
<hudson.model.Queue_-State>
<counter>17</counter>
<items>
<hudson.model.Queue_-BuildableItem>
<actions>
<hudson.model.CauseAction>
<causeBag class="linked-hash-map">
<entry>
<hudson.model.Cause_-UserIdCause/>
<int>1</int>
</entry>
</causeBag>
</hudson.model.CauseAction>
</actions>
<id>17</id>
<task class="hudson.model.FreeStyleProject">k</task>
<inQueueSince>1516354024440</inQueueSince>
<buildableStartMilliseconds>1516354024440</buildableStartMilliseconds>
<isPending>false</isPending>
</hudson.model.Queue_-BuildableItem>
<hudson.model.Queue_-BuildableItem>
<actions>
<hudson.model.CauseAction>
<causeBag class="linked-hash-map">
<entry>
<hudson.model.Cause_-UserIdCause/>
<int>1</int>
</entry>
</causeBag>
</hudson.model.CauseAction>
</actions>
<id>16</id>
<task class="hudson.model.FreeStyleProject">j</task>
<inQueueSince>1516354003647</inQueueSince>
<buildableStartMilliseconds>1516354003648</buildableStartMilliseconds>
<isPending>false</isPending>
</hudson.model.Queue_-BuildableItem>
<hudson.model.Queue_-BuildableItem>
<actions>
<hudson.model.CauseAction>
<causeBag class="linked-hash-map">
<entry>
<hudson.model.Cause_-UserIdCause/>
<int>1</int>
</entry>
</causeBag>
</hudson.model.CauseAction>
</actions>
<id>15</id>
<task class="hudson.model.FreeStyleProject">i</task>
<inQueueSince>1516354001887</inQueueSince>
<buildableStartMilliseconds>1516354001888</buildableStartMilliseconds>
<isPending>false</isPending>
</hudson.model.Queue_-BuildableItem>
<hudson.model.Queue_-BuildableItem>
<actions>
<hudson.model.CauseAction>
<causeBag class="linked-hash-map">
<entry>
<hudson.model.Cause_-UserIdCause/>
<int>1</int>
</entry>
</causeBag>
</hudson.model.CauseAction>
</actions>
<id>14</id>
<task class="hudson.model.FreeStyleProject">h</task>
<inQueueSince>1516353999191</inQueueSince>
<buildableStartMilliseconds>1516353999192</buildableStartMilliseconds>
<isPending>false</isPending>
</hudson.model.Queue_-BuildableItem>
<hudson.model.Queue_-BuildableItem>
<actions>
<hudson.model.CauseAction>
<causeBag class="linked-hash-map">
<entry>
<hudson.model.Cause_-UserIdCause/>
<int>1</int>
</entry>
</causeBag>
</hudson.model.CauseAction>
</actions>
<id>13</id>
<task class="hudson.model.FreeStyleProject">g</task>
<inQueueSince>1516353998031</inQueueSince>
<buildableStartMilliseconds>1516353998032</buildableStartMilliseconds>
<isPending>false</isPending>
</hudson.model.Queue_-BuildableItem>
<hudson.model.Queue_-BuildableItem>
<actions>
<hudson.model.CauseAction>
<causeBag class="linked-hash-map">
<entry>
<hudson.model.Cause_-UserIdCause/>
<int>1</int>
</entry>
</causeBag>
</hudson.model.CauseAction>
</actions>
<id>12</id>
<task class="hudson.model.FreeStyleProject">f</task>
<inQueueSince>1516353996504</inQueueSince>
<buildableStartMilliseconds>1516353996504</buildableStartMilliseconds>
<isPending>false</isPending>
</hudson.model.Queue_-BuildableItem>
<hudson.model.Queue_-BuildableItem>
<actions>
<hudson.model.CauseAction>
<causeBag class="linked-hash-map">
<entry>
<hudson.model.Cause_-UserIdCause/>
<int>1</int>
</entry>
</causeBag>
</hudson.model.CauseAction>
</actions>
<id>11</id>
<task class="hudson.model.FreeStyleProject">e</task>
<inQueueSince>1516353995471</inQueueSince>
<buildableStartMilliseconds>1516353995472</buildableStartMilliseconds>
<isPending>false</isPending>
</hudson.model.Queue_-BuildableItem>
<hudson.model.Queue_-BuildableItem>
<actions>
<hudson.model.CauseAction>
<causeBag class="linked-hash-map">
<entry>
<hudson.model.Cause_-UserIdCause/>
<int>1</int>
</entry>
</causeBag>
</hudson.model.CauseAction>
</actions>
<id>10</id>
<task class="hudson.model.FreeStyleProject">d</task>
<inQueueSince>1516353993872</inQueueSince>
<buildableStartMilliseconds>1516353993873</buildableStartMilliseconds>
<isPending>false</isPending>
</hudson.model.Queue_-BuildableItem>
<hudson.model.Queue_-BuildableItem>
<actions>
<hudson.model.CauseAction>
<causeBag class="linked-hash-map">
<entry>
<hudson.model.Cause_-UserIdCause/>
<int>1</int>
</entry>
</causeBag>
</hudson.model.CauseAction>
</actions>
<id>9</id>
<task class="hudson.model.FreeStyleProject">c</task>
<inQueueSince>1516353991671</inQueueSince>
<buildableStartMilliseconds>1516353991672</buildableStartMilliseconds>
<isPending>false</isPending>
</hudson.model.Queue_-BuildableItem>
<hudson.model.Queue_-BuildableItem>
<actions>
<hudson.model.CauseAction>
<causeBag class="linked-hash-map">
<entry>
<hudson.model.Cause_-UserIdCause/>
<int>1</int>
</entry>
</causeBag>
</hudson.model.CauseAction>
</actions>
<id>8</id>
<task class="hudson.model.FreeStyleProject">b</task>
<inQueueSince>1516353989177</inQueueSince>
<buildableStartMilliseconds>1516353989177</buildableStartMilliseconds>
<isPending>false</isPending>
</hudson.model.Queue_-BuildableItem>
<hudson.model.Queue_-BuildableItem>
<actions>
<hudson.model.CauseAction>
<causeBag class="linked-hash-map">
<entry>
<hudson.model.Cause_-UserIdCause/>
<int>1</int>
</entry>
</causeBag>
</hudson.model.CauseAction>
</actions>
<id>7</id>
<task class="hudson.model.FreeStyleProject">a</task>
<inQueueSince>1516353986148</inQueueSince>
<buildableStartMilliseconds>1516353986151</buildableStartMilliseconds>
<isPending>false</isPending>
</hudson.model.Queue_-BuildableItem>
</items>
</hudson.model.Queue_-State>
\ No newline at end of file
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册