| CODENOTIFIER | HelpYou are not signed inSign in |
Project: StatSVN
Revision: 368
Author: benoitx
Date: 25 Jun 2008 17:23:46
Changes:Fix a few Checkstyle warnings.
Files:| ...... | @@ -47,4 +47,5 @@ | |
| 47 | 47 | import net.sf.statsvn.util.BinaryDiffException; |
| 48 | 48 | import net.sf.statsvn.util.FilenameComparator; |
| 49 | import net.sf.statsvn.util.SvnDiffUtils; | |
| 49 | 50 | import net.sf.statsvn.util.XMLUtil; |
| 50 | 51 | |
| ...... | @@ -65,753 +66,762 @@ | |
| 65 | 66 | */ |
| 66 | 67 | public class SvnLogfileParser { |
| 67 | private static final int INTERMEDIARY_SAVE_INTERVAL_MS = 120000; | |
| 68 | private static final int INTERMEDIARY_SAVE_INTERVAL_MS = 120000; | |
| 68 | 69 | |
| 69 | private static final String REPOSITORIES_XML = "repositories.xml"; | |
| 70 | private static final String REPOSITORIES_XML = "repositories.xml"; | |
| 70 | 71 | |
| 71 | private final SvnLogBuilder builder; | |
| 72 | private final SvnLogBuilder builder; | |
| 72 | 73 | |
| 73 | private final InputStream logFile; | |
| 74 | ||
| 75 | private final RepositoryFileManager repositoryFileManager; | |
| 76 | ||
| 77 | private CacheBuilder cacheBuilder; | |
| 78 | ||
| 79 | private HashSet revsForNewDiff = null; | |
| 80 | ||
| 81 | /** | |
| 82 | * Default Constructor | |
| 83 | * | |
| 84 | * @param repositoryFileManager | |
| 85 | * the repository file manager | |
| 86 | * @param logFile | |
| 87 | * a <tt>Reader</tt> containing the SVN logfile | |
| 88 | * @param builder | |
| 89 | * the builder that will process the log information | |
| 90 | */ | |
| 91 | public SvnLogfileParser(final RepositoryFileManager repositoryFileManager, final InputStream logFile, final SvnLogBuilder builder) { | |
| 92 | this.logFile = logFile; | |
| 93 | this.builder = builder; | |
| 94 | this.repositoryFileManager = repositoryFileManager; | |
| 95 | } | |
| 96 | ||
| 97 | /** | |
| 98 | * Because the log file does not contain the lines added or removed in a | |
| 99 | * commit, and because the logfile contains implicit actions (@link | |
| 100 | * #verifyImplicitActions()), we must query the repository for line | |
| 101 | * differences. This method uses the (@link LineCountsBuilder) to load the | |
| 102 | * persisted information and (@link SvnDiffUtils) to find new information. | |
| 103 | * | |
| 104 | * @param factory | |
| 105 | * the factory used to create SAX parsers. | |
| 106 | * @throws IOException | |
| 107 | */ | |
| 108 | protected void handleLineCounts(final SAXParserFactory factory) throws IOException { | |
| 109 | long startTime = System.currentTimeMillis(); | |
| 110 | final String xmlFile = SvnConfigurationOptions.getCacheDir() + REPOSITORIES_XML; | |
| 111 | ||
| 112 | final RepositoriesBuilder repositoriesBuilder = readAndParseXmlFile(factory, xmlFile); | |
| 113 | cacheFileName = SvnConfigurationOptions.getCacheDir() + repositoriesBuilder.getFileName(repositoryFileManager.getRepositoryUuid()); | |
| 114 | XMLUtil.writeXmlFile(repositoriesBuilder.getDocument(), xmlFile); | |
| 115 | SvnConfigurationOptions.getTaskLogger().log("parsing repositories finished in " + (System.currentTimeMillis() - startTime) + " ms."); | |
| 116 | startTime = System.currentTimeMillis(); | |
| 117 | ||
| 118 | readCache(factory); | |
| 119 | SvnConfigurationOptions.getTaskLogger().log("parsing line counts finished in " + (System.currentTimeMillis() - startTime) + " ms."); | |
| 120 | startTime = System.currentTimeMillis(); | |
| 121 | ||
| 122 | // update the cache xml file with the latest binary status information | |
| 123 | // from the working copy | |
| 124 | cacheBuilder.updateBinaryStatus(builder.getFileBuilders().values(), repositoryFileManager.getRootRevisionNumber()); | |
| 125 | ||
| 126 | final Collection fileBuilders = builder.getFileBuilders().values(); | |
| 127 | ||
| 128 | calculateNumberRequiredCalls(fileBuilders); | |
| 129 | ||
| 130 | // concurrency | |
| 131 | ExecutorService poolService = null; | |
| 132 | if (SvnConfigurationOptions.getNumberSvnDiffThreads() > 1) { | |
| 133 | poolService = Executors.newFixedThreadPool(SvnConfigurationOptions.getNumberSvnDiffThreads()); | |
| 134 | } | |
| 135 | ||
| 136 | boolean isFirstDiff = true; | |
| 137 | calls = 0; | |
| 138 | groupStart = System.currentTimeMillis(); | |
| 139 | boolean poolUseRequired = false; | |
| 140 | ||
| 141 | if (SvnConfigurationOptions.isLegacyDiff()) { | |
| 142 | for (final Iterator iter = fileBuilders.iterator(); iter.hasNext();) { | |
| 143 | final FileBuilder fileBuilder = (FileBuilder) iter.next(); | |
| 144 | final String fileName = fileBuilder.getName(); | |
| 145 | if (fileBuilder.isBinary() || !builder.matchesPatterns(fileName)) { | |
| 146 | continue; | |
| 147 | } | |
| 148 | final List revisions = fileBuilder.getRevisions(); | |
| 149 | for (int i = 0; i < revisions.size(); i++) { | |
| 150 | // line diffs are expensive operations. therefore, the | |
| 151 | // result is | |
| 152 | // stored in the | |
| 153 | // cacheBuilder and eventually persisted in the cache xml | |
| 154 | // file. | |
| 155 | // the next time | |
| 156 | // the file is read the line diffs (or 0/0 in case of binary | |
| 157 | // files) are intialized | |
| 158 | // in the RevisionData. this cause hasNoLines to be false | |
| 159 | // which | |
| 160 | // in turn causes the | |
| 161 | // if clause below to be skipped. | |
| 162 | if (i + 1 < revisions.size() && ((RevisionData) revisions.get(i)).hasNoLines() && !((RevisionData) revisions.get(i)).isDeletion()) { | |
| 163 | if (((RevisionData) revisions.get(i + 1)).isDeletion()) { | |
| 164 | continue; | |
| 165 | } | |
| 166 | final String revNrNew = ((RevisionData) revisions.get(i)).getRevisionNumber(); | |
| 167 | if (cacheBuilder.isBinary(fileName, revNrNew)) { | |
| 168 | continue; | |
| 169 | } | |
| 170 | final String revNrOld = ((RevisionData) revisions.get(i + 1)).getRevisionNumber(); | |
| 171 | ||
| 172 | if (isFirstDiff) { | |
| 173 | SvnConfigurationOptions.getTaskLogger().info("Contacting server to obtain line count information."); | |
| 174 | SvnConfigurationOptions.getTaskLogger().info( | |
| 175 | "This information will be cached so that the next time you run StatSVN, results will be returned more quickly."); | |
| 176 | ||
| 177 | if (SvnConfigurationOptions.isLegacyDiff()) { | |
| 178 | SvnConfigurationOptions.getTaskLogger().info("Using the legacy Subversion 1.3 diff mechanism: one diff per file per revision."); | |
| 179 | } else { | |
| 180 | SvnConfigurationOptions.getTaskLogger().info("Using the Subversion 1.4 diff mechanism: one diff per revision."); | |
| 181 | } | |
| 182 | ||
| 183 | isFirstDiff = false; | |
| 184 | } | |
| 185 | ||
| 186 | final DiffTask diff = new DiffTask(fileName, revNrNew, revNrOld, fileBuilder); | |
| 187 | ||
| 188 | // SvnConfigurationOptions.getTaskLogger().log(Thread.currentThread().getName() | |
| 189 | // + " Schedule task for " + fileName + " rev:" + | |
| 190 | // revNrNew); | |
| 191 | ||
| 192 | poolUseRequired = executeTask(poolService, poolUseRequired, diff); | |
| 193 | } | |
| 194 | } | |
| 195 | } | |
| 196 | } else { | |
| 197 | for (final Iterator iter = revsForNewDiff.iterator(); iter.hasNext();) { | |
| 198 | final String revNrNew = (String) iter.next(); | |
| 199 | final PerRevDiffTask diff = new PerRevDiffTask(revNrNew, builder.getFileBuilders()); | |
| 200 | ||
| 201 | poolUseRequired = executeTask(poolService, poolUseRequired, diff); | |
| 202 | } | |
| 203 | ||
| 204 | } | |
| 205 | waitForPoolIfRequired(poolService); | |
| 206 | SvnConfigurationOptions.getTaskLogger().log("parsing svn diff"); | |
| 207 | XMLUtil.writeXmlFile(cacheBuilder.getDocument(), cacheFileName); | |
| 208 | SvnConfigurationOptions.getTaskLogger().log("parsing svn diff finished in " + (System.currentTimeMillis() - startTime) + " ms."); | |
| 209 | } | |
| 210 | ||
| 211 | private boolean executeTask(final ExecutorService poolService, boolean poolUseRequired, final DiffTask diff) { | |
| 212 | if (poolUseRequired && SvnConfigurationOptions.getNumberSvnDiffThreads() > 1) { | |
| 213 | poolService.execute(diff); | |
| 214 | } else { | |
| 215 | final long start = System.currentTimeMillis(); | |
| 216 | diff.run(); | |
| 217 | final long end = System.currentTimeMillis(); | |
| 218 | poolUseRequired = (end - start) > SvnConfigurationOptions.getThresholdInMsToUseConcurrency(); | |
| 219 | } | |
| 220 | return poolUseRequired; | |
| 221 | } | |
| 222 | ||
| 223 | private void waitForPoolIfRequired(final ExecutorService poolService) { | |
| 224 | if (SvnConfigurationOptions.getNumberSvnDiffThreads() > 1 && poolService != null) { | |
| 225 | SvnConfigurationOptions.getTaskLogger().info( | |
| 226 | "Scheduled " + requiredDiffCalls + " svn diff calls on " + Math.min(requiredDiffCalls, SvnConfigurationOptions.getNumberSvnDiffThreads()) | |
| 227 | + " threads."); | |
| 228 | poolService.shutdown(); | |
| 229 | try { | |
| 230 | SvnConfigurationOptions.getTaskLogger().log("================ Wait for completion ========================="); | |
| 231 | if (!poolService.awaitTermination(2, TimeUnit.DAYS)) { | |
| 232 | SvnConfigurationOptions.getTaskLogger().log("================ TIME OUT!!! ========================="); | |
| 233 | } | |
| 234 | } catch (final InterruptedException e) { | |
| 235 | SvnConfigurationOptions.getTaskLogger().error(e.toString()); | |
| 236 | } | |
| 237 | } | |
| 238 | } | |
| 239 | ||
| 240 | private void calculateNumberRequiredCalls(final Collection fileBuilders) { | |
| 241 | // Calculate the number of required calls... | |
| 242 | requiredDiffCalls = 0; | |
| 243 | ||
| 244 | if (!SvnConfigurationOptions.isLegacyDiff()) { | |
| 245 | revsForNewDiff = new HashSet(); | |
| 246 | } | |
| 247 | ||
| 248 | for (final Iterator iter = fileBuilders.iterator(); iter.hasNext();) { | |
| 249 | final FileBuilder fileBuilder = (FileBuilder) iter.next(); | |
| 250 | final String fileName = fileBuilder.getName(); | |
| 251 | if (!fileBuilder.isBinary() && builder.matchesPatterns(fileName)) { | |
| 252 | final List revisions = fileBuilder.getRevisions(); | |
| 253 | for (int i = 0; i < revisions.size(); i++) { | |
| 254 | if (i + 1 < revisions.size() && ((RevisionData) revisions.get(i)).hasNoLines() && !((RevisionData) revisions.get(i)).isDeletion()) { | |
| 255 | if (((RevisionData) revisions.get(i + 1)).isDeletion()) { | |
| 256 | continue; | |
| 257 | } | |
| 258 | final String revNrNew = ((RevisionData) revisions.get(i)).getRevisionNumber(); | |
| 259 | if (cacheBuilder.isBinary(fileName, revNrNew)) { | |
| 260 | continue; | |
| 261 | } | |
| 262 | // count if legacy diff or this rev wasn't already | |
| 263 | // counted. | |
| 264 | if (revsForNewDiff == null || !revsForNewDiff.contains(revNrNew)) { | |
| 265 | requiredDiffCalls++; | |
| 266 | ||
| 267 | if (revsForNewDiff != null) { | |
| 268 | revsForNewDiff.add(revNrNew); | |
| 269 | } | |
| 270 | } | |
| 271 | } | |
| 272 | } | |
| 273 | } | |
| 274 | } | |
| 275 | // END Calculate the number of required calls... | |
| 276 | } | |
| 277 | ||
| 278 | private void readCache(final SAXParserFactory factory) throws IOException { | |
| 279 | cacheBuilder = new CacheBuilder(builder, repositoryFileManager); | |
| 280 | FileInputStream cacheFile = null; | |
| 281 | try { | |
| 282 | cacheFile = new FileInputStream(cacheFileName); | |
| 283 | final SAXParser parser = factory.newSAXParser(); | |
| 284 | parser.parse(cacheFile, new SvnXmlCacheFileHandler(cacheBuilder)); | |
| 285 | cacheFile.close(); | |
| 286 | } catch (final ParserConfigurationException e) { | |
| 287 | SvnConfigurationOptions.getTaskLogger().error("Cache: " + e.toString()); | |
| 288 | } catch (final SAXException e) { | |
| 289 | SvnConfigurationOptions.getTaskLogger().error("Cache: " + e.toString()); | |
| 290 | } catch (final FileNotFoundException e) { | |
| 291 | SvnConfigurationOptions.getTaskLogger().log("Cache: " + e.toString()); | |
| 292 | } catch (final IOException e) { | |
| 293 | SvnConfigurationOptions.getTaskLogger().error("Cache: " + e.toString()); | |
| 294 | } finally { | |
| 295 | if (cacheFile != null) { | |
| 296 | cacheFile.close(); | |
| 297 | } | |
| 298 | } | |
| 299 | } | |
| 300 | ||
| 301 | private RepositoriesBuilder readAndParseXmlFile(final SAXParserFactory factory, final String xmlFile) throws IOException { | |
| 302 | final RepositoriesBuilder repositoriesBuilder = new RepositoriesBuilder(); | |
| 303 | FileInputStream repositoriesFile = null; | |
| 304 | try { | |
| 305 | repositoriesFile = new FileInputStream(xmlFile); | |
| 306 | final SAXParser parser = factory.newSAXParser(); | |
| 307 | parser.parse(repositoriesFile, new SvnXmlRepositoriesFileHandler(repositoriesBuilder)); | |
| 308 | repositoriesFile.close(); | |
| 309 | } catch (final ParserConfigurationException e) { | |
| 310 | SvnConfigurationOptions.getTaskLogger().error("Repositories: " + e.toString()); | |
| 311 | } catch (final SAXException e) { | |
| 312 | SvnConfigurationOptions.getTaskLogger().error("Repositories: " + e.toString()); | |
| 313 | } catch (final FileNotFoundException e) { | |
| 314 | SvnConfigurationOptions.getTaskLogger().log("Repositories: " + e.toString()); | |
| 315 | } catch (final IOException e) { | |
| 316 | SvnConfigurationOptions.getTaskLogger().error("Repositories: " + e.toString()); | |
| 317 | } finally { | |
| 318 | if (repositoriesFile != null) { | |
| 319 | repositoriesFile.close(); | |
| 320 | } | |
| 321 | } | |
| 322 | return repositoriesBuilder; | |
| 323 | } | |
| 324 | ||
| 325 | /** | |
| 326 | * Parses the logfile. After <tt>parse()</tt> has finished, the result of | |
| 327 | * the parsing process can be obtained from the builder. | |
| 328 | * | |
| 329 | * @throws LogSyntaxException | |
| 330 | * if syntax errors in log | |
| 331 | * @throws IOException | |
| 332 | * if errors while reading from the log Reader | |
| 333 | */ | |
| 334 | public void parse() throws LogSyntaxException, IOException { | |
| 335 | ||
| 336 | final SAXParserFactory factory = parseSvnLog(); | |
| 337 | ||
| 338 | verifyImplicitActions(); | |
| 339 | ||
| 340 | // must be after verifyImplicitActions(); | |
| 341 | removeDirectories(); | |
| 342 | ||
| 343 | handleLineCounts(factory); | |
| 344 | ||
| 345 | } | |
| 346 | ||
| 347 | /** | |
| 348 | * The svn log can contain deletions of directories which imply that all of | |
| 349 | * its contents have been deleted. | |
| 350 | * | |
| 351 | * Furthermore, the svn log can contain entries which are copies from other | |
| 352 | * directories (additions or replacements; I haven't seen modifications with | |
| 353 | * this property, but am not 100% sure) meaning that all files from the | |
| 354 | * other directory are copied here. We currently do not go back through | |
| 355 | * copies, so we must infer what files <i>could</i> have been added during | |
| 356 | * those copies. | |
| 357 | * | |
| 358 | */ | |
| 359 | protected void verifyImplicitActions() { | |
| 360 | // this method most certainly has issues with implicit actions on root | |
| 361 | // folder. | |
| 362 | ||
| 363 | final long startTime = System.currentTimeMillis(); | |
| 364 | SvnConfigurationOptions.getTaskLogger().log("verifying implicit actions ..."); | |
| 365 | ||
| 366 | final HashSet implicitActions = new HashSet(); | |
| 367 | ||
| 368 | // get all filenames | |
| 369 | final ArrayList files = new ArrayList(); | |
| 370 | final Collection fileBuilders = fetchAllFileNames(files); | |
| 371 | ||
| 372 | // sort them so that folders are immediately followed by the folder | |
| 373 | // entries and then by other files which are prefixed by the folder | |
| 374 | // name. | |
| 375 | Collections.sort(files, new FilenameComparator()); | |
| 376 | ||
| 377 | // for each file | |
| 378 | for (int i = 0; i < files.size(); i++) { | |
| 379 | final String parent = files.get(i).toString(); | |
| 380 | final FileBuilder parentBuilder = (FileBuilder) builder.getFileBuilders().get(parent); | |
| 381 | // check to see if there are files that indicate that parent is a | |
| 382 | // folder. | |
| 383 | for (int j = i + 1; j < files.size() && files.get(j).toString().indexOf(parent + "/") == 0; j++) { | |
| 384 | // we might not know that it was a folder. | |
| 385 | repositoryFileManager.addDirectory(parent); | |
| 386 | ||
| 387 | final String child = files.get(j).toString(); | |
| 388 | final FileBuilder childBuilder = (FileBuilder) builder.getFileBuilders().get(child); | |
| 389 | // for all revisions in the the parent folder | |
| 390 | for (final Iterator iter = parentBuilder.getRevisions().iterator(); iter.hasNext();) { | |
| 391 | final RevisionData parentData = (RevisionData) iter.next(); | |
| 392 | int parentRevision; | |
| 393 | try { | |
| 394 | parentRevision = Integer.parseInt(parentData.getRevisionNumber()); | |
| 395 | } catch (final Exception e) { | |
| 396 | continue; | |
| 397 | } | |
| 398 | ||
| 399 | // ignore modifications to folders | |
| 400 | if (parentData.isCreationOrRestore() || parentData.isDeletion()) { | |
| 401 | int k; | |
| 402 | ||
| 403 | // check to see if the parent revision is an implicit | |
| 404 | // action acting on the child. | |
| 405 | k = detectActionOnChildGivenActionOnParent(childBuilder, parentRevision); | |
| 406 | ||
| 407 | // we found something to insert | |
| 408 | if (k < childBuilder.getRevisions().size()) { | |
| 409 | createImplicitAction(implicitActions, child, childBuilder, parentData, k); | |
| 410 | } | |
| 411 | } | |
| 412 | } | |
| 413 | } | |
| 414 | } | |
| 415 | ||
| 416 | // Some implicit revisions may have resulted in double deletion | |
| 417 | // (e.g. deleting a directory and THEN deleting the parent directory). | |
| 418 | // this will get rid of any consecutive deletion. | |
| 419 | cleanPotentialDuplicateImplicitActions(fileBuilders); | |
| 420 | ||
| 421 | // in the preceeding block, we add implicit additions to too may files. | |
| 422 | // possibly a folder was deleted and restored later on, without the | |
| 423 | // specific file being re-added. we get rid of those here. however, | |
| 424 | // without knowledge of what was copied during the implicit additions / | |
| 425 | // replacements, we will remove as many implicit actions as possible | |
| 426 | // | |
| 427 | // this solution is imperfect. | |
| 428 | ||
| 429 | // Examples: | |
| 430 | // IA ID IA ID M A -> ID M A | |
| 431 | // IA ID A D M A -> ID A D M A | |
| 432 | removePotentialInconsistencies(implicitActions, fileBuilders); | |
| 433 | SvnConfigurationOptions.getTaskLogger().log("verifying implicit actions finished in " + (System.currentTimeMillis() - startTime) + " ms."); | |
| 434 | } | |
| 435 | ||
| 436 | private void createImplicitAction(final HashSet implicitActions, final String child, final FileBuilder childBuilder, final RevisionData parentData, | |
| 437 | final int k) { | |
| 438 | // we want to memorize this implicit action. | |
| 439 | final RevisionData implicit = parentData.createCopy(); | |
| 440 | implicitActions.add(implicit); | |
| 441 | ||
| 442 | // avoid concurrent modification errors. | |
| 443 | final List toMove = new ArrayList(); | |
| 444 | for (final Iterator it = childBuilder.getRevisions().subList(k, childBuilder.getRevisions().size()).iterator(); it.hasNext();) { | |
| 445 | final RevisionData revToMove = (RevisionData) it.next(); | |
| 446 | // if | |
| 447 | // (!revToMove.getRevisionNumber().equals(implicit.getRevisionNumber())) | |
| 448 | // { | |
| 449 | toMove.add(revToMove); | |
| 450 | // } | |
| 451 | } | |
| 452 | ||
| 453 | // remove the revisions to be moved. | |
| 454 | childBuilder.getRevisions().removeAll(toMove); | |
| 455 | ||
| 456 | // don't call addRevision directly. buildRevision | |
| 457 | // does more. | |
| 458 | builder.buildFile(child, false, false, new HashMap(), new HashMap()); | |
| 459 | ||
| 460 | // only add the implicit if the last one for the | |
| 461 | // file is NOT a deletion! | |
| 462 | // if (!toMove.isEmpty() && !((RevisionData) | |
| 463 | // toMove.get(0)).isDeletion()) { | |
| 464 | builder.buildRevision(implicit); | |
| 465 | // } | |
| 466 | ||
| 467 | // copy back the revisions we removed. | |
| 468 | for (final Iterator it = toMove.iterator(); it.hasNext();) { | |
| 469 | builder.buildRevision((RevisionData) it.next()); | |
| 470 | } | |
| 471 | } | |
| 472 | ||
| 473 | private int detectActionOnChildGivenActionOnParent(final FileBuilder childBuilder, final int parentRevision) { | |
| 474 | int k; | |
| 475 | for (k = 0; k < childBuilder.getRevisions().size(); k++) { | |
| 476 | final RevisionData childData = (RevisionData) childBuilder.getRevisions().get(k); | |
| 477 | final int childRevision = Integer.parseInt(childData.getRevisionNumber()); | |
| 478 | ||
| 479 | // we don't want to add duplicate entries for the | |
| 480 | // same revision | |
| 481 | if (parentRevision == childRevision) { | |
| 482 | k = childBuilder.getRevisions().size(); | |
| 483 | break; | |
| 484 | } | |
| 485 | ||
| 486 | if (parentRevision > childRevision) { | |
| 487 | break; // we must insert it here! | |
| 488 | } | |
| 489 | } | |
| 490 | return k; | |
| 491 | } | |
| 492 | ||
| 493 | private void removePotentialInconsistencies(final HashSet implicitActions, final Collection fileBuilders) { | |
| 494 | for (final Iterator iter = fileBuilders.iterator(); iter.hasNext();) { | |
| 495 | final FileBuilder filebuilder = (FileBuilder) iter.next(); | |
| 496 | ||
| 497 | // make sure our attic is well set, with our new deletions that we | |
| 498 | // might have added. | |
| 499 | if (!repositoryFileManager.existsInWorkingCopy(filebuilder.getName())) { | |
| 500 | builder.addToAttic(filebuilder.getName()); | |
| 501 | } | |
| 502 | ||
| 503 | // do we detect an inconsistency? | |
| 504 | if (!repositoryFileManager.existsInWorkingCopy(filebuilder.getName()) && !filebuilder.finalRevisionIsDead()) { | |
| 505 | int earliestDelete = -1; | |
| 506 | for (int i = 0; i < filebuilder.getRevisions().size(); i++) { | |
| 507 | final RevisionData data = (RevisionData) filebuilder.getRevisions().get(i); | |
| 508 | ||
| 509 | if (data.isDeletion()) { | |
| 510 | earliestDelete = i; | |
| 511 | } | |
| 512 | ||
| 513 | if ((!data.isCreationOrRestore() && data.isChange()) || !implicitActions.contains(data)) { | |
| 514 | break; | |
| 515 | } | |
| 516 | } | |
| 517 | ||
| 518 | if (earliestDelete > 0) { | |
| 519 | // avoid concurrent modification errors. | |
| 520 | final List toRemove = new ArrayList(); | |
| 521 | for (final Iterator it = filebuilder.getRevisions().subList(0, earliestDelete).iterator(); it.hasNext();) { | |
| 522 | toRemove.add(it.next()); | |
| 523 | } | |
| 524 | filebuilder.getRevisions().removeAll(toRemove); | |
| 525 | } | |
| 526 | } | |
| 527 | } | |
| 528 | } | |
| 529 | ||
| 530 | private void cleanPotentialDuplicateImplicitActions(final Collection fileBuilders) { | |
| 531 | for (final Iterator iter = fileBuilders.iterator(); iter.hasNext();) { | |
| 532 | final FileBuilder filebuilder = (FileBuilder) iter.next(); | |
| 533 | ||
| 534 | boolean previousIsDelete = false; | |
| 535 | final List toRemove = new ArrayList(); | |
| 536 | // for this file, iterate through all revisions and store any | |
| 537 | // deletion revision that follows | |
| 538 | // a deletion. | |
| 539 | for (final Iterator it = filebuilder.getRevisions().iterator(); it.hasNext();) { | |
| 540 | final RevisionData data = (RevisionData) it.next(); | |
| 541 | if (data.isDeletion() && previousIsDelete) { | |
| 542 | toRemove.add(data); | |
| 543 | } | |
| 544 | previousIsDelete = data.isDeletion(); | |
| 545 | } | |
| 546 | ||
| 547 | // get rid of the duplicate deletion for this file. | |
| 548 | if (!toRemove.isEmpty()) { | |
| 549 | filebuilder.getRevisions().removeAll(toRemove); | |
| 550 | } | |
| 551 | } | |
| 552 | } | |
| 553 | ||
| 554 | private Collection fetchAllFileNames(final ArrayList files) { | |
| 555 | final Collection fileBuilders = builder.getFileBuilders().values(); | |
| 556 | for (final Iterator iter = fileBuilders.iterator(); iter.hasNext();) { | |
| 557 | final FileBuilder fileBuilder = (FileBuilder) iter.next(); | |
| 558 | files.add(fileBuilder.getName()); | |
| 559 | } | |
| 560 | return fileBuilders; | |
| 561 | } | |
| 562 | ||
| 563 | /** | |
| 564 | * We have created FileBuilders for directories because we needed the | |
| 565 | * information to be able to find implicit actions. However, we don't want | |
| 566 | * to query directories for their line counts later on. Therefore, we must | |
| 567 | * remove them here. | |
| 568 | * | |
| 569 | * (@link SvnInfoUtils#isDirectory(String)) is used to know what files are | |
| 570 | * directories. Deleted directories are assumed to have been added in (@link | |
| 571 | * #verifyImplicitActions()) | |
| 572 | */ | |
| 573 | protected void removeDirectories() { | |
| 574 | final Collection fileBuilders = builder.getFileBuilders().values(); | |
| 575 | final ArrayList toRemove = new ArrayList(); | |
| 576 | for (final Iterator iter = fileBuilders.iterator(); iter.hasNext();) { | |
| 577 | final FileBuilder fileBuilder = (FileBuilder) iter.next(); | |
| 578 | if (repositoryFileManager.isDirectory(fileBuilder.getName())) { | |
| 579 | toRemove.add(fileBuilder.getName()); | |
| 580 | } | |
| 581 | } | |
| 582 | ||
| 583 | for (final Iterator iter = toRemove.iterator(); iter.hasNext();) { | |
| 584 | builder.getFileBuilders().remove(iter.next()); | |
| 585 | } | |
| 586 | ||
| 587 | } | |
| 588 | ||
| 589 | /** | |
| 590 | * Parses the svn log file. | |
| 591 | * | |
| 592 | * @return the SaxParserFactory, so that it can be reused. | |
| 593 | * @throws IOException | |
| 594 | * errors while reading file. | |
| 595 | * @throws LogSyntaxException | |
| 596 | * invalid log syntax. | |
| 597 | */ | |
| 598 | protected SAXParserFactory parseSvnLog() throws IOException, LogSyntaxException { | |
| 599 | final long startTime = System.currentTimeMillis(); | |
| 600 | SvnConfigurationOptions.getTaskLogger().log("starting to parse..."); | |
| 601 | ||
| 602 | final SAXParserFactory factory = SAXParserFactory.newInstance(); | |
| 603 | try { | |
| 604 | final SAXParser parser = factory.newSAXParser(); | |
| 605 | parser.parse(logFile, new SvnXmlLogFileHandler(builder, repositoryFileManager)); | |
| 606 | } catch (final ParserConfigurationException e) { | |
| 607 | throw new LogSyntaxException("svn log: " + e.getMessage()); | |
| 608 | } catch (final SAXException e) { | |
| 609 | throw new LogSyntaxException("svn log: " + e.getMessage()); | |
| 610 | } | |
| 611 | ||
| 612 | SvnConfigurationOptions.getTaskLogger().log("parsing svn log finished in " + (System.currentTimeMillis() - startTime) + " ms."); | |
| 613 | return factory; | |
| 614 | } | |
| 615 | ||
| 616 | private long totalTime = 0; | |
| 617 | ||
| 618 | private long groupStart = 0; | |
| 619 | ||
| 620 | private int calls = 0; | |
| 621 | ||
| 622 | private int requiredDiffCalls = 0; | |
| 623 | ||
| 624 | private String cacheFileName; | |
| 625 | ||
| 626 | protected class DiffTask implements Runnable { | |
| 627 | protected String fileName; | |
| 628 | ||
| 629 | protected String newRevision; | |
| 630 | ||
| 631 | protected String oldRevision; | |
| 632 | ||
| 633 | protected FileBuilder fileBuilder; | |
| 634 | ||
| 635 | protected DiffTask() { | |
| 636 | } | |
| 637 | ||
| 638 | public DiffTask(final String fileName, final String newRevision, final String oldRevision, final FileBuilder fileBuilder) { | |
| 639 | super(); | |
| 640 | this.fileName = fileName; | |
| 641 | this.newRevision = newRevision; | |
| 642 | this.oldRevision = oldRevision; | |
| 643 | this.fileBuilder = fileBuilder; | |
| 644 | } | |
| 645 | ||
| 646 | /** | |
| 647 | * @return the fileName | |
| 648 | */ | |
| 649 | public String getFileName() { | |
| 650 | return fileName; | |
| 651 | } | |
| 652 | ||
| 653 | /** | |
| 654 | * @param fileName | |
| 655 | * the fileName to set | |
| 656 | */ | |
| 657 | public void setFileName(final String fileName) { | |
| 658 | this.fileName = fileName; | |
| 659 | } | |
| 660 | ||
| 661 | /** | |
| 662 | * @return the newRevision | |
| 663 | */ | |
| 664 | public String getNewRevision() { | |
| 665 | return newRevision; | |
| 666 | } | |
| 667 | ||
| 668 | /** | |
| 669 | * @param newRevision | |
| 670 | * the newRevision to set | |
| 671 | */ | |
| 672 | public void setNewRevision(final String newRevision) { | |
| 673 | this.newRevision = newRevision; | |
| 674 | } | |
| 675 | ||
| 676 | /** | |
| 677 | * @return the oldRevision | |
| 678 | */ | |
| 679 | public String getOldRevision() { | |
| 680 | return oldRevision; | |
| 681 | } | |
| 682 | ||
| 683 | /** | |
| 684 | * @param oldRevision | |
| 685 | * the oldRevision to set | |
| 686 | */ | |
| 687 | public void setOldRevision(final String oldRevision) { | |
| 688 | this.oldRevision = oldRevision; | |
| 689 | } | |
| 690 | ||
| 691 | public void run() { | |
| 692 | int[] lineDiff; | |
| 693 | long end = 0L; | |
| 694 | try { | |
| 695 | // SvnConfigurationOptions.getTaskLogger().log(Thread.currentThread().getName() | |
| 696 | // + " Starts... now"); | |
| 697 | final long start = System.currentTimeMillis(); | |
| 698 | lineDiff = repositoryFileManager.getLineDiff(oldRevision, newRevision, fileName); | |
| 699 | end = System.currentTimeMillis(); | |
| 700 | synchronized (cacheBuilder) { | |
| 701 | totalTime += (end - start); | |
| 702 | } | |
| 703 | ||
| 704 | SvnConfigurationOptions.getTaskLogger().info( | |
| 705 | "svn diff " + (++calls) + "/" + requiredDiffCalls + ": " + fileName + ", r" + oldRevision + " to r" + newRevision + ", +" + lineDiff[0] | |
| 706 | + " -" + lineDiff[1] + " (" + (end - start) + " ms.) " + Thread.currentThread().getName()); | |
| 707 | } catch (final BinaryDiffException e) { | |
| 708 | calls++; | |
| 709 | trackBinaryFile(); | |
| 710 | return; | |
| 711 | } catch (final IOException e) { | |
| 712 | SvnConfigurationOptions.getTaskLogger() | |
| 713 | .error("" + (++calls) + "/" + requiredDiffCalls + " IOException: Unable to obtain diff: " + e.toString()); | |
| 714 | return; | |
| 715 | } | |
| 716 | ||
| 717 | trackFileDiff(lineDiff); | |
| 718 | ||
| 719 | performIntermediarySave(end); | |
| 720 | } | |
| 721 | ||
| 722 | protected void trackBinaryFile() { | |
| 723 | // file is binary and has been deleted | |
| 724 | cacheBuilder.newRevision(fileName, newRevision, "0", "0", true); | |
| 725 | fileBuilder.setBinary(true); | |
| 726 | } | |
| 727 | ||
| 728 | protected void trackFileDiff(final int[] lineDiff) { | |
| 729 | if (lineDiff[0] != -1 && lineDiff[1] != -1) { | |
| 730 | builder.updateRevision(fileName, newRevision, lineDiff[0], lineDiff[1]); | |
| 731 | cacheBuilder.newRevision(fileName, newRevision, lineDiff[0] + "", lineDiff[1] + "", false); | |
| 732 | } else { | |
| 733 | SvnConfigurationOptions.getTaskLogger().info("unknown behaviour; to be investigated:" + fileName + " r:" + oldRevision + "/r:" + newRevision); | |
| 734 | } | |
| 735 | } | |
| 736 | ||
| 737 | protected void performIntermediarySave(long end) { | |
| 738 | synchronized (cacheBuilder) { | |
| 739 | if (end - groupStart > INTERMEDIARY_SAVE_INTERVAL_MS) { | |
| 740 | final long start = System.currentTimeMillis(); | |
| 741 | XMLUtil.writeXmlFile(cacheBuilder.getDocument(), cacheFileName); | |
| 742 | groupStart = System.currentTimeMillis(); | |
| 743 | final double estimateLeftInMs = ((double) totalTime / (double) calls * (requiredDiffCalls - calls) / SvnConfigurationOptions | |
| 744 | .getNumberSvnDiffThreads()); | |
| 745 | end = System.currentTimeMillis(); | |
| 746 | SvnConfigurationOptions.getTaskLogger().info( | |
| 747 | System.getProperty("line.separator") + new Date() + " Intermediary save took " + (end - start) + " ms. Estimated completion=" | |
| 748 | + new Date(end + (long) estimateLeftInMs) + System.getProperty("line.separator")); | |
| 749 | } | |
| 750 | } | |
| 751 | } | |
| 752 | ||
| 753 | } | |
| 754 | ||
| 755 | protected class PerRevDiffTask extends DiffTask { | |
| 756 | protected Map fileBuilders; | |
| 757 | ||
| 758 | public PerRevDiffTask(final String newRevision, final Map fileBuilders) { | |
| 759 | this.newRevision = newRevision; | |
| 760 | this.fileBuilders = fileBuilders; | |
| 761 | } | |
| 762 | ||
| 763 | public void run() { | |
| 764 | int[] lineDiff; | |
| 765 | Vector results; | |
| 766 | long end = 0L; | |
| 767 | try { | |
| 768 | // SvnConfigurationOptions.getTaskLogger().log(Thread.currentThread().getName() | |
| 769 | // + " Starts... now"); | |
| 770 | final long start = System.currentTimeMillis(); | |
| 771 | results = repositoryFileManager.getRevisionDiff(newRevision); | |
| 772 | end = System.currentTimeMillis(); | |
| 773 | synchronized (cacheBuilder) { | |
| 774 | totalTime += (end - start); | |
| 775 | } | |
| 776 | ||
| 777 | SvnConfigurationOptions.getTaskLogger().info( | |
| 778 | "svn diff " + (++calls) + "/" + requiredDiffCalls + " on r" + newRevision + " (" + (end - start) + " ms.) " | |
| 779 | + Thread.currentThread().getName()); | |
| 780 | ||
| 781 | for (int i = 0; i < results.size(); i++) { | |
| 782 | final Object[] element = (Object[]) results.get(i); | |
| 783 | ||
| 784 | if (element.length == 3 && fileBuilders.containsKey(element[0].toString())) { | |
| 785 | fileName = element[0].toString(); | |
| 786 | fileBuilder = (FileBuilder) fileBuilders.get(fileName); | |
| 787 | lineDiff = (int[]) element[1]; | |
| 788 | oldRevision = "?"; | |
| 789 | ||
| 790 | final Boolean isBinary = (Boolean) element[2]; | |
| 791 | if (isBinary.booleanValue()) { | |
| 792 | trackBinaryFile(); | |
| 793 | } | |
| 794 | ||
| 795 | SvnConfigurationOptions.getTaskLogger().info("\t " + fileName + ", on r" + newRevision + ", +" + lineDiff[0] + " -" + lineDiff[1]); | |
| 796 | ||
| 797 | trackFileDiff(lineDiff); | |
| 798 | } else { | |
| 799 | SvnConfigurationOptions.getTaskLogger().error("Problem with diff " + i + " for revision " + newRevision + "."); | |
| 800 | } | |
| 801 | } | |
| 802 | ||
| 803 | } catch (final BinaryDiffException e) { | |
| 804 | // not supposed to happen. tracked individually. | |
| 805 | return; | |
| 806 | } catch (final IOException e) { | |
| 807 | SvnConfigurationOptions.getTaskLogger() | |
| 808 | .error("" + (++calls) + "/" + requiredDiffCalls + " IOException: Unable to obtain diff: " + e.toString()); | |
| 809 | return; | |
| 810 | } | |
| 811 | ||
| 812 | performIntermediarySave(end); | |
| 813 | } | |
| 814 | ||
| 815 | } | |
| 74 | private final InputStream logFile; | |
| 816 | 75 | |
| 76 | private final RepositoryFileManager repositoryFileManager; | |
| 77 | ||
| 78 | private CacheBuilder cacheBuilder; | |
| 79 | ||
| 80 | private HashSet revsForNewDiff = null; | |
| 81 | ||
| 82 | /** | |
| 83 | * Default Constructor | |
| 84 | * | |
| 85 | * @param repositoryFileManager | |
| 86 | * the repository file manager | |
| 87 | * @param logFile | |
| 88 | * a <tt>Reader</tt> containing the SVN logfile | |
| 89 | * @param builder | |
| 90 | * the builder that will process the log information | |
| 91 | */ | |
| 92 | public SvnLogfileParser(final RepositoryFileManager repositoryFileManager, final InputStream logFile, final SvnLogBuilder builder) { | |
| 93 | this.logFile = logFile; | |
| 94 | this.builder = builder; | |
| 95 | this.repositoryFileManager = repositoryFileManager; | |
| 96 | } | |
| 97 | ||
| 98 | /** | |
| 99 | * Because the log file does not contain the lines added or removed in a | |
| 100 | * commit, and because the logfile contains implicit actions (@link | |
| 101 | * #verifyImplicitActions()), we must query the repository for line | |
| 102 | * differences. This method uses the (@link LineCountsBuilder) to load the | |
| 103 | * persisted information and (@link SvnDiffUtils) to find new information. | |
| 104 | * | |
| 105 | * @param factory | |
| 106 | * the factory used to create SAX parsers. | |
| 107 | * @throws IOException | |
| 108 | */ | |
| 109 | protected void handleLineCounts(final SAXParserFactory factory) throws IOException { | |
| 110 | long startTime = System.currentTimeMillis(); | |
| 111 | final String xmlFile = SvnConfigurationOptions.getCacheDir() + REPOSITORIES_XML; | |
| 112 | ||
| 113 | final RepositoriesBuilder repositoriesBuilder = readAndParseXmlFile(factory, xmlFile); | |
| 114 | cacheFileName = SvnConfigurationOptions.getCacheDir() + repositoriesBuilder.getFileName(repositoryFileManager.getRepositoryUuid()); | |
| 115 | XMLUtil.writeXmlFile(repositoriesBuilder.getDocument(), xmlFile); | |
| 116 | SvnConfigurationOptions.getTaskLogger().log("parsing repositories finished in " + (System.currentTimeMillis() - startTime) + " ms."); | |
| 117 | startTime = System.currentTimeMillis(); | |
| 118 | ||
| 119 | readCache(factory); | |
| 120 | SvnConfigurationOptions.getTaskLogger().log("parsing line counts finished in " + (System.currentTimeMillis() - startTime) + " ms."); | |
| 121 | startTime = System.currentTimeMillis(); | |
| 122 | ||
| 123 | // update the cache xml file with the latest binary status information | |
| 124 | // from the working copy | |
| 125 | cacheBuilder.updateBinaryStatus(builder.getFileBuilders().values(), repositoryFileManager.getRootRevisionNumber()); | |
| 126 | ||
| 127 | final Collection fileBuilders = builder.getFileBuilders().values(); | |
| 128 | ||
| 129 | calculateNumberRequiredCalls(fileBuilders); | |
| 130 | ||
| 131 | // concurrency | |
| 132 | ExecutorService poolService = null; | |
| 133 | if (SvnConfigurationOptions.getNumberSvnDiffThreads() > 1) { | |
| 134 | poolService = Executors.newFixedThreadPool(SvnConfigurationOptions.getNumberSvnDiffThreads()); | |
| 135 | } | |
| 136 | ||
| 137 | boolean isFirstDiff = true; | |
| 138 | calls = 0; | |
| 139 | groupStart = System.currentTimeMillis(); | |
| 140 | boolean poolUseRequired = false; | |
| 141 | ||
| 142 | if (SvnConfigurationOptions.isLegacyDiff()) { | |
| 143 | for (final Iterator iter = fileBuilders.iterator(); iter.hasNext();) { | |
| 144 | final FileBuilder fileBuilder = (FileBuilder) iter.next(); | |
| 145 | final String fileName = fileBuilder.getName(); | |
| 146 | if (fileBuilder.isBinary() || !builder.matchesPatterns(fileName)) { | |
| 147 | continue; | |
| 148 | } | |
| 149 | final List revisions = fileBuilder.getRevisions(); | |
| 150 | for (int i = 0; i < revisions.size(); i++) { | |
| 151 | // line diffs are expensive operations. therefore, the | |
| 152 | // result is | |
| 153 | // stored in the | |
| 154 | // cacheBuilder and eventually persisted in the cache xml | |
| 155 | // file. | |
| 156 | // the next time | |
| 157 | // the file is read the line diffs (or 0/0 in case of binary | |
| 158 | // files) are intialized | |
| 159 | // in the RevisionData. this cause hasNoLines to be false | |
| 160 | // which | |
| 161 | // in turn causes the | |
| 162 | // if clause below to be skipped. | |
| 163 | if (i + 1 < revisions.size() && ((RevisionData) revisions.get(i)).hasNoLines() && !((RevisionData) revisions.get(i)).isDeletion()) { | |
| 164 | if (((RevisionData) revisions.get(i + 1)).isDeletion()) { | |
| 165 | continue; | |
| 166 | } | |
| 167 | final String revNrNew = ((RevisionData) revisions.get(i)).getRevisionNu |