From 84756b54a1665076456883f0c2e9beea943c4dc0 Mon Sep 17 00:00:00 2001 From: Anatoly Kislov Date: 2017年8月30日 12:46:15 +0300 Subject: [PATCH 1/2] wip: separate RepoHasher and CommitHasher, basic integration of CodeLongevity --- .../kotlin/app/{ => hashers}/CodeLongevity.kt | 92 ++++++++++------- .../CommitHasher.kt} | 89 ++++------------- src/main/kotlin/app/hashers/RepoHasher.kt | 98 +++++++++++++++++++ src/main/kotlin/app/ui/UpdateRepoState.kt | 9 +- src/test/CommitUploadProtocolTest.kt | 13 +-- 5 files changed, 183 insertions(+), 118 deletions(-) rename src/main/kotlin/app/{ => hashers}/CodeLongevity.kt (75%) rename src/main/kotlin/app/{RepoHasher.kt => hashers/CommitHasher.kt} (73%) create mode 100644 src/main/kotlin/app/hashers/RepoHasher.kt diff --git a/src/main/kotlin/app/CodeLongevity.kt b/src/main/kotlin/app/hashers/CodeLongevity.kt similarity index 75% rename from src/main/kotlin/app/CodeLongevity.kt rename to src/main/kotlin/app/hashers/CodeLongevity.kt index b7ec21d7..537a043e 100644 --- a/src/main/kotlin/app/CodeLongevity.kt +++ b/src/main/kotlin/app/hashers/CodeLongevity.kt @@ -1,14 +1,18 @@ // Copyright 2017 Sourcerer Inc. All Rights Reserved. // Author: Alexander Surkov (alex@sourcerer.io) -package app - +package app.hashers + +import app.Logger +import app.api.Api +import app.config.Configurator +import app.model.LocalRepo +import app.model.Repo +import app.utils.RepoHelper import org.eclipse.jgit.diff.DiffFormatter import org.eclipse.jgit.diff.DiffEntry import org.eclipse.jgit.diff.RawText -import org.eclipse.jgit.internal.storage.file.FileRepository -import org.eclipse.jgit.lib.ObjectId -import org.eclipse.jgit.lib.ObjectLoader +import org.eclipse.jgit.api.Git import org.eclipse.jgit.lib.Repository import org.eclipse.jgit.revwalk.RevCommit import org.eclipse.jgit.revwalk.RevWalk @@ -28,7 +32,8 @@ class RevCommitLine(val commit: RevCommit, val file: String, val line: Int) * * TODO(Alex): the text arg is solely for testing proposes (remove it) */ -class CodeLine(val from: RevCommitLine, val to: RevCommitLine, val text: String) { +class CodeLine(val from: RevCommitLine, val to: RevCommitLine, + val text: String) { // TODO(alex): oldId and newId may be computed as a hash built from commit, // file name and line number, if we are going to send the data outside a @@ -69,10 +74,14 @@ class CodeLine(val from: RevCommitLine, val to: RevCommitLine, val text: String) /** * Used to compute age of code lines in the repo. */ -class CodeLongevity(repoPath: String, tailRev: String) { - val repo = FileRepository(repoPath) +class CodeLongevity(private val localRepo: LocalRepo, + private val serverRepo: Repo, + private val api: Api, + private val configurator: Configurator, + private val git: Git, tailRev: String = "") { + val repo: Repository = git.repository val head: RevCommit = - RevWalk(repo).parseCommit(repo.resolve("refs/heads/master")) + RevWalk(repo).parseCommit(repo.resolve(RepoHelper.MASTER_BRANCH)) val tail: RevCommit? = if (tailRev != "") RevWalk(repo).parseCommit(repo.resolve(tailRev)) else null @@ -83,30 +92,31 @@ class CodeLongevity(repoPath: String, tailRev: String) { */ var codeLines: MutableList = mutableListOf() - init { + fun update() { compute() - } - // TODO(alex) debugging, remove it - fun ohNoDoesItReallyWork(email: String) { - var sum: Long = 0 - var total: Long = 0 + // TODO(anatoly): Add emails from server or hashAll. + val emails = hashSetOf(localRepo.author.email) + + val sum: MutableMap = emails.associate { Pair(it, 0L) } + .toMutableMap() + val total: Int = codeLines.size for (line in codeLines) { - val author = line.from.commit.getAuthorIdent() - if (author.getEmailAddress() != email) { + val email = line.from.commit.authorIdent.emailAddress + if (!emails.contains(email)) { continue } - println(line.toString()) - println(" Age: ${line.age} secs") - sum += line.age - total++ - } + Logger.debug(line.toString()) + Logger.debug("Age: ${line.age} secs") - //println("All lines:") - //codeLines.forEach { line -> line.printme() } + sum[email] = sum[email]!! + line.age + } - var avg = if (total> 0) sum / total else 0 - println("avg code line age for <$email> is ${avg} seconds, lines total: ${total}") + for (email in emails) { + val avg = if (total> 0) sum[email]!! / total else 0 + println("Average code line age for <$email> is $avg seconds, " + + "lines total: $total") + } } /** @@ -143,14 +153,16 @@ class CodeLongevity(repoPath: String, tailRev: String) { var commit: RevCommit? = revWalk.next() // move the walker to the head while (commit != null && commit != tail) { - var parentCommit: RevCommit? = revWalk.next() + val parentCommit: RevCommit? = revWalk.next() - println("commit: ${commit.getName()}; '${commit.getShortMessage()}'") + Logger.debug("commit: ${commit.getName()}; " + + "'${commit.getShortMessage()}'") if (parentCommit != null) { - println("parent commit: ${parentCommit.getName()}; '${parentCommit.getShortMessage()}'") + Logger.debug("parent commit: ${parentCommit.getName()}; " + + "'${parentCommit.getShortMessage()}'") } else { - println("parent commit: null") + Logger.debug("parent commit: null") } // A step back in commits history. Update the files map according @@ -161,7 +173,7 @@ class CodeLongevity(repoPath: String, tailRev: String) { val oldId = diff.getOldId().toObjectId() val newPath = diff.getNewPath() val newId = diff.getNewId().toObjectId() - println("old: '$oldPath', new: '$newPath'") + Logger.debug("old: '$oldPath', new: '$newPath'") // Skip binary files. var fileId = if (newPath != DiffEntry.DEV_NULL) newId else oldId @@ -186,8 +198,12 @@ class CodeLongevity(repoPath: String, tailRev: String) { } // If a file was deleted, then the new path is /dev/null. - val path = if (newPath != DiffEntry.DEV_NULL) newPath else oldPath - var lines = files.get(path)!! + val path = if (newPath != DiffEntry.DEV_NULL) { + newPath + } else { + oldPath + } + val lines = files.get(path)!! // Update the lines array to match the diff's edit list changes. // Traverse the edit list backwards to keep indices of the edit @@ -200,7 +216,8 @@ class CodeLongevity(repoPath: String, tailRev: String) { var insStart = edit.getBeginB() var insEnd = edit.getEndB() val insCount = edit.getLengthB() - println("del ($delStart, $delEnd), ins ($insStart, $insEnd)") + Logger.debug("del ($delStart, $delEnd), " + + "ins ($insStart, $insEnd)") // Deletion case. Chase down the deleted lines through the // history. @@ -209,7 +226,9 @@ class CodeLongevity(repoPath: String, tailRev: String) { for (idx in delStart .. delEnd - 1) { tmpLines.add(RevCommitLine(commit, oldPath, idx)) } - lines.addAll(delStart, tmpLines) + // TODO(alex): Approve code change. + // delStart index caused IndexOutOfBoundException. + lines.addAll(tmpLines) } // Insertion case. Track it. @@ -243,7 +262,8 @@ class CodeLongevity(repoPath: String, tailRev: String) { for ((file, lines) in files) { for (idx in 0 .. lines.size - 1) { val from = RevCommitLine(tail, file, idx) - val cl = CodeLine(from, lines[idx], "no data (too lazy to compute)") + val cl = CodeLine(from, lines[idx], + "no data (too lazy to compute)") codeLines.add(cl) } } diff --git a/src/main/kotlin/app/RepoHasher.kt b/src/main/kotlin/app/hashers/CommitHasher.kt similarity index 73% rename from src/main/kotlin/app/RepoHasher.kt rename to src/main/kotlin/app/hashers/CommitHasher.kt index 65771881..c4d5159f 100644 --- a/src/main/kotlin/app/RepoHasher.kt +++ b/src/main/kotlin/app/hashers/CommitHasher.kt @@ -1,8 +1,9 @@ // Copyright 2017 Sourcerer Inc. All Rights Reserved. // Author: Anatoly Kislov (anatoly@sourcerer.io) -package app +package app.hashers +import app.Logger import app.api.Api import app.config.Configurator import app.extractors.Extractor @@ -17,8 +18,6 @@ import org.eclipse.jgit.api.Git import org.eclipse.jgit.diff.DiffEntry import org.eclipse.jgit.lib.Repository import org.eclipse.jgit.revwalk.RevWalk -import java.io.File -import java.io.IOException import java.nio.charset.Charset import org.eclipse.jgit.diff.DiffFormatter import org.eclipse.jgit.lib.ObjectId @@ -28,42 +27,15 @@ import java.nio.file.Paths import java.util.concurrent.TimeUnit /** - * RepoHasher hashes repository and uploads stats to server. + * CommitHasher hashes repository and uploads stats to server. */ -class RepoHasher(private val localRepo: LocalRepo, private val api: Api, - private val configurator: Configurator) { - private var repo: Repo = Repo() - private val git: Git = loadGit() ?: - throw IllegalStateException("Git failed to load") +class CommitHasher(private val localRepo: LocalRepo, + private val repo: Repo = Repo(), + private val api: Api, + private val configurator: Configurator, + private val git: Git) { private val gitRepo: Repository = git.repository - private fun loadGit(): Git? { - return try { - Git.open(File(localRepo.path)) - } catch (e: IOException) { - Logger.error("Cannot access repository at path " - + "$localRepo.path", e) - null - } - } - - private fun closeGit() { - gitRepo.close() - git.close() - } - - private fun calculateRepoRehashes() { - val initialCommit = getObservableCommits().blockingLast() - repo.initialCommitRehash = initialCommit.rehash - repo.rehash = RepoHelper.calculateRepoRehash(initialCommit.rehash, - localRepo) - } - - private fun isKnownRepo(): Boolean { - return configurator.getRepos() - .find { it.rehash == repo.rehash } != null - } - private fun findFirstOverlappingCommit(): Commit? { val serverHistoryCommits = repo.commits.toHashSet() return getObservableCommits() @@ -153,16 +125,6 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api, } } - private fun getRepoFromServer() { - repo = api.getRepo(repo.rehash) - Logger.debug("Received repo from server with ${repo.commits.size} " + - "commits") - } - - private fun postRepoToServer() { - api.postRepo(repo) - } - private fun postCommitsToServer(commits: List) { if (commits.isNotEmpty()) { api.postCommits(commits) @@ -213,32 +175,15 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api, } fun update() { - if (!RepoHelper.isValidRepo(localRepo.path)) { - Logger.error("Invalid repo $localRepo") - return - } - - println("Hashing $localRepo...") - localRepo.parseGitConfig(gitRepo.config) - repo.userEmail = localRepo.author.email - calculateRepoRehashes() - - if (isKnownRepo()) { - getRepoFromServer() - - // Delete missing commits. If found at least one common commit - // then next commits are not deleted because hash of a commit - // calculated including hashes of its parents. - val firstOverlapCommit = findFirstOverlappingCommit() - val deletedCommits = repo.commits - .takeWhile { it.rehash != firstOverlapCommit?.rehash } - deleteCommitsOnServer(deletedCommits) - } - + // Delete missing local commits. If found at least one common commit + // then next commits are not deleted because hash of a commit + // calculated including hashes of its parents. + val firstOverlapCommit = findFirstOverlappingCommit() + val deletedCommits = repo.commits + .takeWhile { it.rehash != firstOverlapCommit?.rehash } + deleteCommitsOnServer(deletedCommits) + + // Hash added and missing server commits and send them to server. hashAndSendCommits() - postRepoToServer() - - println("Hashing $localRepo successfully finished.") - closeGit() } } diff --git a/src/main/kotlin/app/hashers/RepoHasher.kt b/src/main/kotlin/app/hashers/RepoHasher.kt new file mode 100644 index 00000000..f6d9a759 --- /dev/null +++ b/src/main/kotlin/app/hashers/RepoHasher.kt @@ -0,0 +1,98 @@ +// Copyright 2017 Sourcerer Inc. All Rights Reserved. +// Author: Anatoly Kislov (anatoly@sourcerer.io) + +package app.hashers + +import app.Logger +import app.api.Api +import app.config.Configurator +import app.model.LocalRepo +import app.model.Repo +import app.utils.RepoHelper +import org.eclipse.jgit.api.Git +import org.eclipse.jgit.revwalk.RevCommit +import org.eclipse.jgit.revwalk.RevWalk +import java.io.File +import java.io.IOException + +class RepoHasher(private val localRepo: LocalRepo, private val api: Api, + private val configurator: Configurator) { + var repo: Repo = Repo() + + init { + if (!RepoHelper.isValidRepo(localRepo.path)) { + throw IllegalArgumentException("Invalid repo $localRepo") + } + + println("Hashing $localRepo...") + val git = loadGit(localRepo.path) + try { + localRepo.parseGitConfig(git.repository.config) + initializeRepo(git) + + if (!isKnownRepo()) { + // Notify server about new contributor and his email. + postRepoToServer() + } + // Get repo setup (commits, emails to hash) from server. + getRepoFromServer() + + // Hash by all plugins. + CommitHasher(localRepo, repo, api, configurator, git).update() + CodeLongevity(localRepo, repo, api, configurator, git).update() + + // Confirm hashing completion. + postRepoToServer() + } + finally { + closeGit(git) + } + println("Hashing $localRepo successfully finished.") + } + + private fun loadGit(path: String): Git { + return try { + Git.open(File(path)) + } catch (e: IOException) { + throw IllegalStateException("Cannot access repository at $path") + } + } + + private fun closeGit(git: Git) { + git.repository?.close() + git.close() + } + + private fun isKnownRepo(): Boolean { + return configurator.getRepos() + .find { it.rehash == repo.rehash } != null + } + + private fun getRepoFromServer() { + repo = api.getRepo(repo.rehash) + Logger.debug("Received repo from server with ${repo.commits.size} " + + "commits") + } + + private fun postRepoToServer() { + api.postRepo(repo) + } + + private fun initializeRepo(git: Git) { + repo = Repo(userEmail = localRepo.author.email) + repo.initialCommitRehash = getInitialCommitRehash(git) + repo.rehash = RepoHelper.calculateRepoRehash(repo.initialCommitRehash, + localRepo) + } + + private fun getInitialCommitRehash(git: Git): String { + val head: RevCommit = RevWalk(git.repository) + .parseCommit(git.repository.resolve(RepoHelper.MASTER_BRANCH)) + + val revWalk = RevWalk(git.repository) + revWalk.markStart(head) + + val initialCommit = revWalk.last() + return initialCommit.id.name + } +} diff --git a/src/main/kotlin/app/ui/UpdateRepoState.kt b/src/main/kotlin/app/ui/UpdateRepoState.kt index 27966e99..c4a45859 100644 --- a/src/main/kotlin/app/ui/UpdateRepoState.kt +++ b/src/main/kotlin/app/ui/UpdateRepoState.kt @@ -3,8 +3,8 @@ package app.ui +import app.hashers.RepoHasher import app.Logger -import app.RepoHasher import app.api.Api import app.config.Configurator import app.utils.RequestException @@ -20,10 +20,11 @@ class UpdateRepoState constructor(private val context: Context, println("Hashing your git repositories.") for (repo in configurator.getLocalRepos()) { try { - RepoHasher(repo, api, configurator).update() + RepoHasher(repo, api, configurator) } catch (e: RequestException) { - Logger.error("Network error while hashing $repo, " - + "skipping...", e) + Logger.error("Network error while hashing $repo", e) + } catch (e: Exception) { + Logger.error("Error while hashing $repo", e) } } println("The repositories have been hashed. See result online on your " diff --git a/src/test/CommitUploadProtocolTest.kt b/src/test/CommitUploadProtocolTest.kt index 8f7e3be1..59b91466 100644 --- a/src/test/CommitUploadProtocolTest.kt +++ b/src/test/CommitUploadProtocolTest.kt @@ -3,7 +3,7 @@ package test -import app.RepoHasher +import app.hashers.CommitHasher import app.api.MockApi import app.config.MockConfigurator import app.model.* @@ -20,6 +20,7 @@ import kotlin.test.assertNotEquals class CommitUploadProtocolTest : Spek({ val repoPath = "./tmp_repo/.git" val git = Git.init().setGitDir(File(repoPath)).call() + val gitHasher = Git.open(File(repoPath)) val localRepo: LocalRepo = LocalRepo(repoPath) val initialCommit = Commit(git.commit().setMessage("Initial commit.").call()) val repoRehash = RepoHelper.calculateRepoRehash(initialCommit.rehash, localRepo) @@ -47,7 +48,7 @@ class CommitUploadProtocolTest : Spek({ val mockApi = MockApi(mockRepo = repo) val mockConfigurator = MockConfigurator(mockRepos = mutableListOf(repo)) - RepoHasher(localRepo, mockApi, mockConfigurator).update() + CommitHasher(localRepo, repo, mockApi, mockConfigurator, gitHasher).update() it("doesn't send added commits") { assertEquals(0, mockApi.receivedAddedCommits.size) @@ -65,7 +66,7 @@ class CommitUploadProtocolTest : Spek({ val revCommit = git.commit().setMessage("Second commit.").call() val addedCommit = Commit(revCommit) - RepoHasher(localRepo, mockApi, mockConfigurator).update() + CommitHasher(localRepo, repo, mockApi, mockConfigurator, gitHasher).update() it("doesn't send deleted commits") { assertEquals(0, mockApi.receivedDeletedCommits.size) @@ -96,7 +97,7 @@ class CommitUploadProtocolTest : Spek({ val revCommit = git.commit().setMessage(message).call() authorCommits.add(Commit(revCommit)) } - RepoHasher(localRepo, mockApi, mockConfigurator).update() + CommitHasher(localRepo, repo, mockApi, mockConfigurator, gitHasher).update() it("posts five commits as added") { assertEquals(5, mockApi.receivedAddedCommits.size) @@ -152,14 +153,14 @@ class CommitUploadProtocolTest : Spek({ val revCommit = git.commit().setMessage(message).call() addedCommits.add(Commit(revCommit)) } - RepoHasher(localRepo, mockApi, mockConfigurator).update() + CommitHasher(localRepo, repo, mockApi, mockConfigurator, gitHasher).update() // Remove one commit from server history. val removedCommit = addedCommits.removeAt(1) repo.commits = addedCommits.toList().asReversed() mockConfigurator = MockConfigurator(mockRepos = mutableListOf(repo)) mockApi = MockApi(mockRepo = repo) - RepoHasher(localRepo, mockApi, mockConfigurator).update() + CommitHasher(localRepo, repo, mockApi, mockConfigurator, gitHasher).update() it("adds posts one commit as added and received commit is lost one") { assertEquals(1, mockApi.receivedAddedCommits.size) From 3655555a38ceab80f44a50e5ce233ba5b7734023 Mon Sep 17 00:00:00 2001 From: Anatoly Kislov Date: 2017年8月30日 20:42:28 +0300 Subject: [PATCH 2/2] chore: comments --- src/main/kotlin/app/hashers/CommitHasher.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/app/hashers/CommitHasher.kt b/src/main/kotlin/app/hashers/CommitHasher.kt index c4d5159f..478d3e76 100644 --- a/src/main/kotlin/app/hashers/CommitHasher.kt +++ b/src/main/kotlin/app/hashers/CommitHasher.kt @@ -175,9 +175,9 @@ class CommitHasher(private val localRepo: LocalRepo, } fun update() { - // Delete missing local commits. If found at least one common commit - // then next commits are not deleted because hash of a commit - // calculated including hashes of its parents. + // Delete locally missing commits from server. If found at least one + // common commit then next commits are not deleted because hash of a + // commit calculated including hashes of its parents. val firstOverlapCommit = findFirstOverlappingCommit() val deletedCommits = repo.commits .takeWhile { it.rehash != firstOverlapCommit?.rehash }

AltStyle によって変換されたページ (->オリジナル) /