--- /dev/null
+#!/bin/bash
+# --------------------------------------------------------------------------
+# This is for migrating Tickets from one service to another.
+#
+# usage:
+#
+# migrate-tickets.sh <outputservice> <baseFolder>
+#
+# --------------------------------------------------------------------------
+
+if [[ -z $1 || -z $2 ]]; then
+ echo "Please specify the output ticket service and your baseFolder!";
+ echo "";
+ echo "usage:";
+ echo " migrate-tickets <outputservice> <baseFolder>";
+ echo "";
+ exit 1;
+fi
+
+java -cp gitblit.jar:./ext/* com.gitblit.MigrateTickets $1 --baseFolder $2
+
#
# --------------------------------------------------------------------------
-if [ -z $1 ]; then
+if [[ -z $1 ]]; then
echo "Please specify your baseFolder!";
echo "";
echo "usage:";
--- /dev/null
+@REM --------------------------------------------------------------------------\r
+@REM This is for migrating Tickets from one service to another.\r
+@REM\r
+@REM usage:\r
+@REM migrate-tickets <outputservice> <baseFolder>\r
+@REM\r
+@REM --------------------------------------------------------------------------\r
+@if [%1]==[] goto help\r
+\r
+@if [%2]==[] goto help\r
+\r
+@java -cp gitblit.jar;"%CD%\ext\*" com.gitblit.MigrateTickets %1 --baseFolder %2\r
+@goto end\r
+\r
+:help\r
+@echo "Please specify the output ticket service and your baseFolder!"\r
+@echo\r
+@echo " migrate-tickets com.gitblit.tickets.RedisTicketService c:/gitblit-data"\r
+@echo\r
+\r
+:end
\ No newline at end of file
--- /dev/null
+/*
+ * Copyright 2014 gitblit.com.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gitblit;
+
+import java.io.File;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.kohsuke.args4j.Argument;
+import org.kohsuke.args4j.CmdLineException;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
+
+import com.gitblit.manager.IRepositoryManager;
+import com.gitblit.manager.IRuntimeManager;
+import com.gitblit.manager.RepositoryManager;
+import com.gitblit.manager.RuntimeManager;
+import com.gitblit.models.RepositoryModel;
+import com.gitblit.models.TicketModel;
+import com.gitblit.models.TicketModel.Change;
+import com.gitblit.tickets.BranchTicketService;
+import com.gitblit.tickets.FileTicketService;
+import com.gitblit.tickets.ITicketService;
+import com.gitblit.tickets.RedisTicketService;
+import com.gitblit.utils.StringUtils;
+
+/**
+ * A command-line tool to move all tickets from one ticket service to another.
+ *
+ * @author James Moger
+ *
+ */
+public class MigrateTickets {
+
+ public static void main(String... args) {
+ MigrateTickets migrate = new MigrateTickets();
+
+ // filter out the baseFolder parameter
+ List<String> filtered = new ArrayList<String>();
+ String folder = "data";
+ for (int i = 0; i < args.length; i++) {
+ String arg = args[i];
+ if (arg.equals("--baseFolder")) {
+ if (i + 1 == args.length) {
+ System.out.println("Invalid --baseFolder parameter!");
+ System.exit(-1);
+ } else if (!".".equals(args[i + 1])) {
+ folder = args[i + 1];
+ }
+ i = i + 1;
+ } else {
+ filtered.add(arg);
+ }
+ }
+
+ Params.baseFolder = folder;
+ Params params = new Params();
+ CmdLineParser parser = new CmdLineParser(params);
+ try {
+ parser.parseArgument(filtered);
+ if (params.help) {
+ migrate.usage(parser, null);
+ return;
+ }
+ } catch (CmdLineException t) {
+ migrate.usage(parser, t);
+ return;
+ }
+
+ // load the settings
+ FileSettings settings = params.FILESETTINGS;
+ if (!StringUtils.isEmpty(params.settingsfile)) {
+ if (new File(params.settingsfile).exists()) {
+ settings = new FileSettings(params.settingsfile);
+ }
+ }
+
+ // migrate tickets
+ migrate.migrate(new File(Params.baseFolder), settings, params.outputServiceName);
+ System.exit(0);
+ }
+
+ /**
+ * Display the command line usage of MigrateTickets.
+ *
+ * @param parser
+ * @param t
+ */
+ protected final void usage(CmdLineParser parser, CmdLineException t) {
+ System.out.println(Constants.BORDER);
+ System.out.println(Constants.getGitBlitVersion());
+ System.out.println(Constants.BORDER);
+ System.out.println();
+ if (t != null) {
+ System.out.println(t.getMessage());
+ System.out.println();
+ }
+ if (parser != null) {
+ parser.printUsage(System.out);
+ System.out
+ .println("\nExample:\n java -gitblit.jar com.gitblit.MigrateTickets com.gitblit.tickets.RedisTicketService --baseFolder c:\\gitblit-data");
+ }
+ System.exit(0);
+ }
+
+ /**
+ * Migrate all tickets
+ *
+ * @param baseFolder
+ * @param settings
+ * @param outputServiceName
+ */
+ protected void migrate(File baseFolder, IStoredSettings settings, String outputServiceName) {
+ // disable some services
+ settings.overrideSetting(Keys.web.allowLuceneIndexing, false);
+ settings.overrideSetting(Keys.git.enableGarbageCollection, false);
+ settings.overrideSetting(Keys.git.enableMirroring, false);
+ settings.overrideSetting(Keys.web.activityCacheDays, 0);
+ settings.overrideSetting(ITicketService.SETTING_UPDATE_DIFFSTATS, false);
+
+ IRuntimeManager runtimeManager = new RuntimeManager(settings, baseFolder).start();
+ IRepositoryManager repositoryManager = new RepositoryManager(runtimeManager, null).start();
+
+ String inputServiceName = settings.getString(Keys.tickets.service, BranchTicketService.class.getSimpleName());
+ if (StringUtils.isEmpty(inputServiceName)) {
+ System.err.println(MessageFormat.format("Please define a ticket service in \"{0}\"", Keys.tickets.service));
+ System.exit(1);
+ }
+
+ ITicketService inputService = null;
+ ITicketService outputService = null;
+ try {
+ inputService = getService(inputServiceName, runtimeManager, repositoryManager);
+ outputService = getService(outputServiceName, runtimeManager, repositoryManager);
+ } catch (Exception e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+
+ if (!inputService.isReady()) {
+ System.err.println(String.format("%s INPUT service is not ready, check config.", inputService.getClass().getSimpleName()));
+ System.exit(1);
+ }
+
+ if (!outputService.isReady()) {
+ System.err.println(String.format("%s OUTPUT service is not ready, check config.", outputService.getClass().getSimpleName()));
+ System.exit(1);
+ }
+
+ // migrate tickets
+ long start = System.nanoTime();
+ long totalTickets = 0;
+ long totalChanges = 0;
+ for (RepositoryModel repository : repositoryManager.getRepositoryModels(null)) {
+ Set<Long> ids = inputService.getIds(repository);
+ if (ids == null || ids.isEmpty()) {
+ // nothing to migrate
+ continue;
+ }
+
+ // delete any tickets we may have in the output ticket service
+ outputService.deleteAll(repository);
+
+ for (long id : ids) {
+ List<Change> journal = inputService.getJournal(repository, id);
+ if (journal == null || journal.size() == 0) {
+ continue;
+ }
+ TicketModel ticket = outputService.createTicket(repository, id, journal.get(0));
+ if (ticket == null) {
+ System.err.println(String.format("Failed to migrate %s #%s", repository.name, id));
+ System.exit(1);
+ }
+ totalTickets++;
+ System.out.println(String.format("%s #%s: %s", repository.name, ticket.number, ticket.title));
+ for (int i = 1; i < journal.size(); i++) {
+ TicketModel updated = outputService.updateTicket(repository, ticket.number, journal.get(i));
+ if (updated != null) {
+ System.out.println(String.format(" applied change %d", i));
+ totalChanges++;
+ } else {
+ System.err.println(String.format("Failed to apply change %d:\n%s", i, journal.get(i)));
+ System.exit(1);
+ }
+ }
+ }
+ }
+
+ inputService.stop();
+ outputService.stop();
+
+ repositoryManager.stop();
+ runtimeManager.stop();
+
+ long end = System.nanoTime();
+
+ System.out.println(String.format("Migrated %d tickets composed of %d journal entries in %d seconds",
+ totalTickets, totalTickets + totalChanges, TimeUnit.NANOSECONDS.toSeconds(end - start)));
+ }
+
+ protected ITicketService getService(String serviceName, IRuntimeManager runtimeManager, IRepositoryManager repositoryManager) throws Exception {
+ ITicketService service = null;
+ Class<?> serviceClass = Class.forName(serviceName);
+ if (RedisTicketService.class.isAssignableFrom(serviceClass)) {
+ // Redis ticket service
+ service = new RedisTicketService(runtimeManager, null, null, null, repositoryManager).start();
+ } else if (BranchTicketService.class.isAssignableFrom(serviceClass)) {
+ // Branch ticket service
+ service = new BranchTicketService(runtimeManager, null, null, null, repositoryManager).start();
+ } else if (FileTicketService.class.isAssignableFrom(serviceClass)) {
+ // File ticket service
+ service = new FileTicketService(runtimeManager, null, null, null, repositoryManager).start();
+ } else {
+ System.err.println("Unknown ticket service " + serviceName);
+ }
+ return service;
+ }
+
+ /**
+ * Parameters.
+ */
+ public static class Params {
+
+ public static String baseFolder;
+
+ @Option(name = "--help", aliases = { "-h"}, usage = "Show this help")
+ public Boolean help = false;
+
+ private final FileSettings FILESETTINGS = new FileSettings(new File(baseFolder, Constants.PROPERTIES_FILE).getAbsolutePath());
+
+ @Option(name = "--repositoriesFolder", usage = "Git Repositories Folder", metaVar = "PATH")
+ public String repositoriesFolder = FILESETTINGS.getString(Keys.git.repositoriesFolder, "git");
+
+ @Option(name = "--settings", usage = "Path to alternative settings", metaVar = "FILE")
+ public String settingsfile;
+
+ @Argument(index = 0, required = true, metaVar = "OUTPUTSERVICE", usage = "The destination/output ticket service")
+ public String outputServiceName;
+ }
+}
return hasTicket;
}
+ /**
+ * Returns the assigned ticket ids.
+ *
+ * @return the assigned ticket ids
+ */
+ @Override
+ public synchronized Set<Long> getIds(RepositoryModel repository) {
+ Repository db = repositoryManager.getRepository(repository.name);
+ try {
+ if (getTicketsBranch(db) == null) {
+ return Collections.emptySet();
+ }
+ Set<Long> ids = new TreeSet<Long>();
+ List<PathModel> paths = JGitUtils.getDocuments(db, Arrays.asList("json"), BRANCH);
+ for (PathModel path : paths) {
+ String name = path.name.substring(path.name.lastIndexOf('/') + 1);
+ if (!JOURNAL.equals(name)) {
+ continue;
+ }
+ String tid = path.path.split("/")[2];
+ long ticketId = Long.parseLong(tid);
+ ids.add(ticketId);
+ }
+ return ids;
+ } finally {
+ if (db != null) {
+ db.close();
+ }
+ }
+ }
+
/**
* Assigns a new ticket id.
*
}
AtomicLong lastId = lastAssignedId.get(repository.name);
if (lastId.get() <= 0) {
- List<PathModel> paths = JGitUtils.getDocuments(db, Arrays.asList("json"), BRANCH);
- for (PathModel path : paths) {
- String name = path.name.substring(path.name.lastIndexOf('/') + 1);
- if (!JOURNAL.equals(name)) {
- continue;
- }
- String tid = path.path.split("/")[2];
- long ticketId = Long.parseLong(tid);
- if (ticketId > lastId.get()) {
- lastId.set(ticketId);
+ Set<Long> ids = getIds(repository);
+ for (long id : ids) {
+ if (id > lastId.get()) {
+ lastId.set(id);
}
}
}
}
}
+ /**
+ * Retrieves the journal for the ticket.
+ *
+ * @param repository
+ * @param ticketId
+ * @return a journal, if it exists, otherwise null
+ */
+ @Override
+ protected List<Change> getJournalImpl(RepositoryModel repository, long ticketId) {
+ Repository db = repositoryManager.getRepository(repository.name);
+ try {
+ List<Change> changes = getJournal(db, ticketId);
+ if (ArrayUtils.isEmpty(changes)) {
+ log.warn("Empty journal for {}:{}", repository, ticketId);
+ return null;
+ }
+ return changes;
+ } finally {
+ db.close();
+ }
+ }
+
/**
* Returns the journal for the specified ticket.
*
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
return hasTicket;
}
+ @Override
+ public synchronized Set<Long> getIds(RepositoryModel repository) {
+ Set<Long> ids = new TreeSet<Long>();
+ Repository db = repositoryManager.getRepository(repository.name);
+ try {
+ // identify current highest ticket id by scanning the paths in the tip tree
+ File dir = new File(db.getDirectory(), TICKETS_PATH);
+ dir.mkdirs();
+ List<File> journals = findAll(dir, JOURNAL);
+ for (File journal : journals) {
+ // Reconstruct ticketId from the path
+ // id/26/326/journal.json
+ String path = FileUtils.getRelativePath(dir, journal);
+ String tid = path.split("/")[1];
+ long ticketId = Long.parseLong(tid);
+ ids.add(ticketId);
+ }
+ } finally {
+ if (db != null) {
+ db.close();
+ }
+ }
+ return ids;
+ }
+
/**
* Assigns a new ticket id.
*
}
AtomicLong lastId = lastAssignedId.get(repository.name);
if (lastId.get() <= 0) {
- // identify current highest ticket id by scanning the paths in the tip tree
- File dir = new File(db.getDirectory(), TICKETS_PATH);
- dir.mkdirs();
- List<File> journals = findAll(dir, JOURNAL);
- for (File journal : journals) {
- // Reconstruct ticketId from the path
- // id/26/326/journal.json
- String path = FileUtils.getRelativePath(dir, journal);
- String tid = path.split("/")[1];
- long ticketId = Long.parseLong(tid);
- if (ticketId > lastId.get()) {
- lastId.set(ticketId);
+ Set<Long> ids = getIds(repository);
+ for (long id : ids) {
+ if (id > lastId.get()) {
+ lastId.set(id);
}
}
}
}
/**
- * Retrieves the ticket from the repository by first looking-up the changeId
- * associated with the ticketId.
+ * Retrieves the ticket from the repository.
*
* @param repository
* @param ticketId
}
}
+ /**
+ * Retrieves the journal for the ticket.
+ *
+ * @param repository
+ * @param ticketId
+ * @return a journal, if it exists, otherwise null
+ */
+ @Override
+ protected List<Change> getJournalImpl(RepositoryModel repository, long ticketId) {
+ Repository db = repositoryManager.getRepository(repository.name);
+ try {
+ List<Change> changes = getJournal(db, ticketId);
+ if (ArrayUtils.isEmpty(changes)) {
+ log.warn("Empty journal for {}:{}", repository, ticketId);
+ return null;
+ }
+ return changes;
+ } finally {
+ db.close();
+ }
+ }
+
/**
* Returns the journal for the specified ticket.
*
*/
public abstract class ITicketService {
+ public static final String SETTING_UPDATE_DIFFSTATS = "migration.updateDiffstats";
+
private static final String LABEL = "label";
private static final String MILESTONE = "milestone";
private final Map<String, List<TicketMilestone>> milestonesCache;
+ private final boolean updateDiffstats;
+
private static class TicketKey {
final String repository;
final long ticketId;
this.labelsCache = new ConcurrentHashMap<String, List<TicketLabel>>();
this.milestonesCache = new ConcurrentHashMap<String, List<TicketMilestone>>();
+
+ this.updateDiffstats = settings.getBoolean(SETTING_UPDATE_DIFFSTATS, true);
}
/**
return false;
}
+ /**
+ * Returns the set of assigned ticket ids in the repository.
+ *
+ * @param repository
+ * @return a set of assigned ticket ids in the repository
+ * @since 1.6.0
+ */
+ public abstract Set<Long> getIds(RepositoryModel repository);
+
/**
* Assigns a new ticket id.
*
ticket = getTicketImpl(repository, ticketId);
// if ticket exists
if (ticket != null) {
- if (ticket.hasPatchsets()) {
+ if (ticket.hasPatchsets() && updateDiffstats) {
Repository r = repositoryManager.getRepository(repository.name);
try {
Patchset patchset = ticket.getCurrentPatchset();
*/
protected abstract TicketModel getTicketImpl(RepositoryModel repository, long ticketId);
+
+ /**
+ * Returns the journal used to build a ticket.
+ *
+ * @param repository
+ * @param ticketId
+ * @return the journal for the ticket, if it exists, otherwise null
+ * @since 1.6.0
+ */
+ public final List<Change> getJournal(RepositoryModel repository, long ticketId) {
+ if (hasTicket(repository, ticketId)) {
+ List<Change> journal = getJournalImpl(repository, ticketId);
+ return journal;
+ }
+ return null;
+ }
+
+ /**
+ * Retrieves the ticket journal.
+ *
+ * @param repository
+ * @param ticketId
+ * @return a ticket, if it exists, otherwise null
+ * @since 1.6.0
+ */
+ protected abstract List<Change> getJournalImpl(RepositoryModel repository, long ticketId);
+
/**
* Get the ticket url
*
import java.util.Collections;
import java.util.List;
+import java.util.Set;
import com.gitblit.manager.INotificationManager;
import com.gitblit.manager.IPluginManager;
return false;
}
+ @Override
+ public synchronized Set<Long> getIds(RepositoryModel repository) {
+ return Collections.emptySet();
+ }
+
@Override
public synchronized long assignNewId(RepositoryModel repository) {
return 0L;
return null;
}
+ @Override
+ protected List<Change> getJournalImpl(RepositoryModel repository, long ticketId) {
+ return null;
+ }
+
@Override
public boolean supportsAttachments() {
return false;
import java.util.Collections;
import java.util.List;
import java.util.Set;
+import java.util.TreeSet;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
return false;
}
+ @Override
+ public Set<Long> getIds(RepositoryModel repository) {
+ Set<Long> ids = new TreeSet<Long>();
+ Jedis jedis = pool.getResource();
+ try {// account for migrated tickets
+ Set<String> keys = jedis.keys(key(repository, KeyType.journal, "*"));
+ for (String tkey : keys) {
+ // {repo}:journal:{id}
+ String id = tkey.split(":")[2];
+ long ticketId = Long.parseLong(id);
+ ids.add(ticketId);
+ }
+ } catch (JedisException e) {
+ log.error("failed to assign new ticket id in Redis @ " + getUrl(), e);
+ pool.returnBrokenResource(jedis);
+ jedis = null;
+ } finally {
+ if (jedis != null) {
+ pool.returnResource(jedis);
+ }
+ }
+ return ids;
+ }
+
/**
* Assigns a new ticket id.
*
String key = key(repository, KeyType.counter, null);
String val = jedis.get(key);
if (isNull(val)) {
- jedis.set(key, "0");
+ long lastId = 0;
+ Set<Long> ids = getIds(repository);
+ for (long id : ids) {
+ if (id > lastId) {
+ lastId = id;
+ }
+ }
+ jedis.set(key, "" + lastId);
}
long ticketNumber = jedis.incr(key);
return ticketNumber;
}
/**
- * Retrieves the ticket from the repository by first looking-up the changeId
- * associated with the ticketId.
+ * Retrieves the ticket from the repository.
*
* @param repository
* @param ticketId
return null;
}
+ /**
+ * Retrieves the journal for the ticket.
+ *
+ * @param repository
+ * @param ticketId
+ * @return a journal, if it exists, otherwise null
+ */
+ @Override
+ protected List<Change> getJournalImpl(RepositoryModel repository, long ticketId) {
+ Jedis jedis = pool.getResource();
+ if (jedis == null) {
+ return null;
+ }
+
+ try {
+ List<Change> changes = getJournal(jedis, repository, ticketId);
+ if (ArrayUtils.isEmpty(changes)) {
+ log.warn("Empty journal for {}:{}", repository, ticketId);
+ return null;
+ }
+ return changes;
+ } catch (JedisException e) {
+ log.error("failed to retrieve journal from Redis @ " + getUrl(), e);
+ pool.returnBrokenResource(jedis);
+ jedis = null;
+ } finally {
+ if (jedis != null) {
+ pool.returnResource(jedis);
+ }
+ }
+ return null;
+ }
+
/**
* Returns the journal for the specified ticket.
*
curl --insecure --user admin:admin "https://localhost:8443/rpc?req=reindex_tickets"
curl --insecure --user admin:admin "https://localhost:8443/rpc?req=reindex_tickets&name=gitblit.git"
+#### Migrating Tickets between Ticket Services
+
+##### Gitblit GO
+
+Gitblit GO ships with a script that executes the *com.gitblit.MigrateTickets* tool included in the Gitblit jar file. This tool will migrate *all* tickets in *all* repositories **AND** must be run when Gitblit is offline.
+
+ migrate-tickets <outputservice> <baseFolder>
+
+For example, this would migrate tickets from the current ticket service configured in `c:\gitblit-data\gitblit.properties` to a Redis ticket service. The Redis service is configured in the same config file so you must be sure to properly setup all appropriate Redis settings.
+
+ migrate-tickets com.gitblit.tickets.RedisTicketService c:\gitblit-data
+
+##### Gitblit WAR/Express
+
+Gitblit WAR/Express does not ship with anything other than the WAR, but you can still migrate tickets offline with a little extra effort.
+
+*Windows*
+
+ java -cp "C:/path/to/WEB-INF/lib/*" com.gitblit.MigrateTickets <outputservice> --baseFolder <baseFolder>
+
+*Linux/Unix/Mac OSX*
+
+ java -cp /path/to/WEB-INF/lib/* com.gitblit.MigrateTickets <outputservice> --baseFolder <baseFolder>
+
// query non-existent ticket\r
TicketModel nonExistent = service.getTicket(getRepository(), 0);\r
assertNull(nonExistent);\r
- \r
+\r
// create and insert a ticket\r
Change c1 = newChange("testCreation() " + Long.toHexString(System.currentTimeMillis()));\r
TicketModel ticket = service.createTicket(getRepository(), c1);\r
assertEquals(1, results.size());\r
assertTrue(results.get(0).title.startsWith("testUpdates"));\r
\r
+ // check the ids\r
+ assertEquals("[1, 2]", service.getIds(getRepository()).toString());\r
+\r
// delete all tickets\r
for (TicketModel aTicket : allTickets) {\r
assertTrue(service.deleteTicket(getRepository(), aTicket.number, "D"));\r
projects / gitblit.git / commitdiff