diff --git a/core/src/main/scala/com/nulabinc/backlog/migration/common/client/BacklogAPIClient.scala b/core/src/main/scala/com/nulabinc/backlog/migration/common/client/BacklogAPIClient.scala index 363fd698..1201b3ee 100644 --- a/core/src/main/scala/com/nulabinc/backlog/migration/common/client/BacklogAPIClient.scala +++ b/core/src/main/scala/com/nulabinc/backlog/migration/common/client/BacklogAPIClient.scala @@ -17,6 +17,12 @@ trait BacklogAPIClient extends BacklogClient { def importWiki(params: ImportWikiParams): Wiki + def importDocument(jsonBody: String): String + + def importUpdateDocumentContent(documentId: String, jsonBody: String): Unit + + def importDocumentComment(documentId: String, jsonBody: String): String + def addRateLimitEventListener(listener: RateLimitEventListener): Unit def removeRateLimitEventListener(listener: RateLimitEventListener): Unit diff --git a/core/src/main/scala/com/nulabinc/backlog/migration/common/client/BacklogAPIClientImpl.scala b/core/src/main/scala/com/nulabinc/backlog/migration/common/client/BacklogAPIClientImpl.scala index 852ba634..cb54071c 100644 --- a/core/src/main/scala/com/nulabinc/backlog/migration/common/client/BacklogAPIClientImpl.scala +++ b/core/src/main/scala/com/nulabinc/backlog/migration/common/client/BacklogAPIClientImpl.scala @@ -1,6 +1,15 @@ package com.nulabinc.backlog.migration.common.client +import java.net.http.{ + HttpClient => JHttpClient, + HttpRequest => JHttpRequest, + HttpResponse => JHttpResponse +} +import java.net.{URI, URLEncoder} +import java.nio.charset.StandardCharsets +import java.time.Duration import java.util +import java.util.Date import com.nulabinc.backlog.migration.common.client.params._ import com.nulabinc.backlog.migration.common.conf.BacklogConfiguration @@ -30,6 +39,32 @@ object IAAH { val empty: IAAH = IAAH("") } +private class JsonBacklogHttpResponse(response: JHttpResponse[String]) + extends BacklogHttpResponse { + override def getStatusCode: Int = response.statusCode() + + override def getRateLimitLimit: Int = + response.headers().firstValueAsLong("X-RateLimit-Limit").orElse(0L).toInt + + override def getRateLimitRemaining: Int = + response.headers().firstValueAsLong("X-RateLimit-Remaining").orElse(0L).toInt + + override def getRateLimitResetDate: Date = { + val reset = response.headers().firstValueAsLong("X-RateLimit-Reset") + if (reset.isPresent) new Date(reset.getAsLong * 1000) else null + } + + override def getRateLimitReset: String = + response.headers().firstValue("X-RateLimit-Reset").orElse(null) + + override def asInputStream(): java.io.InputStream = + new java.io.ByteArrayInputStream(response.body().getBytes(StandardCharsets.UTF_8)) + + override def asString(): String = response.body() + + override def getFileNameFromContentDisposition: String = null +} + class BacklogAPIClientImpl(configure: BacklogConfigure, iaah: IAAH) extends BacklogClientImpl(configure, BacklogAPIClientImpl.create) with BacklogAPIClient @@ -76,6 +111,38 @@ class BacklogAPIClientImpl(configure: BacklogConfigure, iaah: IAAH) factory.importWiki(post(buildEndpoint("wikis/import"), params.getParamList, headers)) } + // Document import APIs require a JSON request body (unlike the form-urlencoded + // params used by importWiki/importIssue), which backlog4j's BacklogHttpClient + // cannot send. Talk to these endpoints directly instead. + private val jsonHttpClient: JHttpClient = + JHttpClient + .newBuilder() + .connectTimeout(Duration.ofMillis(configure.getConnectionTimeout)) + .build() + + private def sendJson(method: String, endpoint: String, jsonBody: String): String = { + val uriSeparator = if (endpoint.contains("?")) "&" else "?" + val apiKeyParam = URLEncoder.encode(configure.getApiKey, StandardCharsets.UTF_8.name()) + val request = JHttpRequest + .newBuilder() + .uri(URI.create(s"$endpoint$uriSeparator" + s"apiKey=$apiKeyParam")) + .timeout(Duration.ofMillis(configure.getReadTimeout)) + .header("Content-Type", "application/json; charset=UTF-8") + .header("iaah", iaah.value) + .method(method, JHttpRequest.BodyPublishers.ofString(jsonBody, StandardCharsets.UTF_8)) + .build() + val response = + jsonHttpClient.send(request, JHttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)) + val statusCode = response.statusCode() + if (statusCode < 200 || statusCode >= 300) { + val message = + if (statusCode == rateLimitStatusCode) "The API usage limit has been exceeded." + else "backlog api request failed." + throw new BacklogAPIException(message, new JsonBacklogHttpResponse(response)) + } + response.body() + } + override def importIssue(params: ImportIssueParams): Issue = retryRateLimit() { client.importIssue(params) } @@ -96,6 +163,21 @@ class BacklogAPIClientImpl(configure: BacklogConfigure, iaah: IAAH) client.importWiki(params) } + override def importDocument(jsonBody: String): String = retryRateLimit() { + sendJson("POST", buildEndpoint("documents/import"), jsonBody) + } + + override def importUpdateDocumentContent(documentId: String, jsonBody: String): Unit = + retryRateLimit() { + sendJson("PATCH", buildEndpoint(s"documents/$documentId/content/import"), jsonBody) + () + } + + override def importDocumentComment(documentId: String, jsonBody: String): String = + retryRateLimit() { + sendJson("POST", buildEndpoint(s"documents/$documentId/comments/import"), jsonBody) + } + override def delete( endpoint: String, parameters: util.List[NameValuePair] diff --git a/core/src/main/scala/com/nulabinc/backlog/migration/common/service/DocumentService.scala b/core/src/main/scala/com/nulabinc/backlog/migration/common/service/DocumentService.scala index 5e1c0d76..4d13b40f 100644 --- a/core/src/main/scala/com/nulabinc/backlog/migration/common/service/DocumentService.scala +++ b/core/src/main/scala/com/nulabinc/backlog/migration/common/service/DocumentService.scala @@ -2,7 +2,22 @@ package com.nulabinc.backlog.migration.common.service import java.io.InputStream -import com.nulabinc.backlog.migration.common.domain.{BacklogDocument, BacklogDocumentTree} +import com.nulabinc.backlog.migration.common.domain.{ + BacklogAttachment, + BacklogDocument, + BacklogDocumentComment, + BacklogDocumentTag, + BacklogDocumentTree +} + +// Counts of what rewriteIssueMentions did to the issueMention nodes in one +// document. total = rewritten + skippedExternalProject + unresolved. +final case class IssueMentionRewriteStats( + total: Int, + rewritten: Int, + skippedExternalProject: Int, + unresolved: Int +) /** * @author @@ -23,4 +38,83 @@ trait DocumentService { attachmentId: Long ): Option[(String, InputStream)] + def create( + projectId: Long, + document: BacklogDocument, + optParentId: Option[String], + addLast: Boolean, + isTrash: Boolean, + propertyResolver: PropertyResolver + ): String + + def updateContent( + documentId: String, + document: BacklogDocument, + propertyResolver: PropertyResolver + ): Unit + + def addComment( + documentId: String, + comment: BacklogDocumentComment, + propertyResolver: PropertyResolver + ): Either[Throwable, String] + + def addAttachment( + documentId: String, + path: String + ): Either[Throwable, BacklogAttachment] + + def addTags( + documentId: String, + tagNames: Seq[String] + ): Either[Throwable, Seq[BacklogDocumentTag]] + + // The document body (ProseMirror JSON) anchors each inline comment to a + // range of text via an `inlineComment` mark carrying the comment's id + // (`{"type":"inlineComment","attrs":{"comment":{"id":"...","statusId":...}}}`). + // That id is only valid within the source space, so it must be rewritten to + // the id assigned when the comment was recreated at the destination, + // otherwise the app can't resolve the mark and the comment isn't shown as + // linked to the document. + def rewriteInlineCommentIds( + document: BacklogDocument, + commentIdMap: Map[String, String] + ): BacklogDocument + + // The document body (ProseMirror JSON) can embed an `issueMention` node + // carrying a snapshot of a referenced issue's source-space key, numeric + // id, and project key/id + // (`{"type":"issueMention","attrs":{"id":"PROJ-1","label":"...", + // "mentionType":"inline","projectKey":"PROJ","projectId":1,"issueId":2}}`; + // `issueId`/`projectId` are sometimes absent from real data). Mentions of + // issues in the source project must be rewritten to the key/id assigned + // when that issue was recreated at the destination, and to the + // destination project's key/id, otherwise the app can't resolve the + // mention. Mentions of any other project are left completely untouched, + // since this tool migrates one project per run and has no mapping data + // for other projects. When a same-project mention can't be resolved (the + // issue wasn't found in either map, e.g. it was deleted at the source or + // failed to migrate), the mention is left as-is and a warning is logged + // rather than failing the migration. + // + // The same snapshot is duplicated as a bracket tag in the document's plain + // text mirror (`optPlain`), e.g. `[issueMention id="PROJ-1" label="..." + // mentionType="inline" projectKey="PROJ" projectId="1" issueId="2"]`. Every + // mention actually rewritten in the JSON has its exact old/new tag text + // substituted into `optPlain` too (via literal substring replacement, not + // regex, since labels may contain unescaped `[`/`]`), so the plain-text + // mirror doesn't keep pointing at the source space after migration. If the + // expected old tag text can't be found in `optPlain` (e.g. it drifted from + // the JSON), that mention's plain text is left unchanged and a warning is + // logged, again without failing the migration. + def rewriteIssueMentions( + document: BacklogDocument, + issueIdMap: Map[Long, Long], + issueKeyMap: Map[String, String], + srcProjectId: Long, + srcProjectKey: String, + dstProjectId: Long, + dstProjectKey: String + ): (BacklogDocument, IssueMentionRewriteStats) + } diff --git a/core/src/main/scala/com/nulabinc/backlog/migration/common/service/DocumentServiceImpl.scala b/core/src/main/scala/com/nulabinc/backlog/migration/common/service/DocumentServiceImpl.scala index dcd4f1b1..4f891bf1 100644 --- a/core/src/main/scala/com/nulabinc/backlog/migration/common/service/DocumentServiceImpl.scala +++ b/core/src/main/scala/com/nulabinc/backlog/migration/common/service/DocumentServiceImpl.scala @@ -1,6 +1,6 @@ package com.nulabinc.backlog.migration.common.service -import java.io.InputStream +import java.io.{File, FileInputStream, InputStream} import java.lang.Thread.sleep import javax.inject.Inject @@ -11,15 +11,28 @@ import com.nulabinc.backlog.migration.common.convert.writes.{ DocumentTreeWrites, DocumentWrites } -import com.nulabinc.backlog.migration.common.domain.{BacklogDocument, BacklogDocumentTree} -import com.nulabinc.backlog.migration.common.utils.Logging +import com.nulabinc.backlog.migration.common.domain.{ + BacklogAttachment, + BacklogDocument, + BacklogDocumentComment, + BacklogDocumentCommentReply, + BacklogDocumentTag, + BacklogDocumentTree, + BacklogUser +} +import com.nulabinc.backlog.migration.common.utils.{FileUtil, Logging} import com.nulabinc.backlog4j.api.option.{ + AddDocumentTagsParams, GetDocumentTreeParams, GetDocumentsCountParams, GetDocumentsParams } +import com.nulabinc.backlog4j.internal.file.AttachmentDataImpl +import spray.json.DefaultJsonProtocol._ +import spray.json._ import scala.jdk.CollectionConverters._ +import scala.util.Using /** * @author @@ -93,6 +106,404 @@ class DocumentServiceImpl @Inject() (implicit None } + override def create( + projectId: Long, + document: BacklogDocument, + optParentId: Option[String], + addLast: Boolean, + isTrash: Boolean, + propertyResolver: PropertyResolver + ): String = { + val jsonBody = + createDocumentJson( + projectId, + document, + optParentId, + addLast, + isTrash, + propertyResolver + ).compactPrint + val response = backlog.importDocument(jsonBody) + JsonParser(response).asJsObject.fields("id").convertTo[String] + } + + override def updateContent( + documentId: String, + document: BacklogDocument, + propertyResolver: PropertyResolver + ): Unit = { + val jsonBody = updateContentJson(document, propertyResolver).compactPrint + backlog.importUpdateDocumentContent(documentId, jsonBody) + } + + override def addComment( + documentId: String, + comment: BacklogDocumentComment, + propertyResolver: PropertyResolver + ): Either[Throwable, String] = + try { + val jsonBody = commentJson(comment, propertyResolver).compactPrint + val response = backlog.importDocumentComment(documentId, jsonBody) + Right(JsonParser(response).asJsObject.fields("id").convertTo[String]) + } catch { + case e: Throwable => + logger.error(e.getMessage, e) + Left(e) + } + + def createDocumentJson( + projectId: Long, + document: BacklogDocument, + optParentId: Option[String], + addLast: Boolean, + isTrash: Boolean, + propertyResolver: PropertyResolver + ): JsObject = { + val fields = scala.collection.mutable.Map[String, JsValue]( + "projectId" -> JsNumber(projectId), + "title" -> JsString(document.title), + "addLast" -> JsBoolean(addLast), + "isTrash" -> JsBoolean(isTrash) + ) + document.optEmoji.foreach(emoji => fields += "emoji" -> JsString(emoji)) + optParentId.foreach(parentId => fields += "parentId" -> JsString(parentId)) + document.optCreated.foreach(created => fields += "created" -> JsString(created)) + resolvedUserId(document.optCreatedUser, propertyResolver) + .foreach(id => fields += "createdUserId" -> JsNumber(id)) + document.optUpdated.foreach(updated => fields += "updated" -> JsString(updated)) + resolvedUserId(document.optUpdatedUser, propertyResolver) + .foreach(id => fields += "updatedUserId" -> JsNumber(id)) + JsObject(fields.toMap) + } + + def updateContentJson( + document: BacklogDocument, + propertyResolver: PropertyResolver + ): JsObject = { + val fields = scala.collection.mutable.Map[String, JsValue]( + "json" -> document.optJson.map(_.parseJson).getOrElse(JsObject.empty), + "plain" -> JsString(document.optPlain.getOrElse("")) + ) + document.optUpdated.foreach(updated => fields += "updated" -> JsString(updated)) + resolvedUserId(document.optUpdatedUser, propertyResolver) + .foreach(id => fields += "updatedUserId" -> JsNumber(id)) + JsObject(fields.toMap) + } + + def commentJson( + comment: BacklogDocumentComment, + propertyResolver: PropertyResolver + ): JsObject = { + val fields = scala.collection.mutable.Map[String, JsValue]( + "content" -> JsString(comment.content), + "plain" -> JsString(comment.plain), + "statusId" -> JsNumber(comment.statusId), + "commentType" -> JsString(comment.commentType) + ) + comment.optCreated.foreach(created => fields += "created" -> JsString(created)) + resolvedUserId(comment.optCreatedUser, propertyResolver) + .foreach(id => fields += "createdUserId" -> JsNumber(id)) + comment.optUpdated.foreach(updated => fields += "updated" -> JsString(updated)) + if (comment.replies.nonEmpty) { + fields += "replies" -> JsArray( + comment.replies.map(replyFields(_, propertyResolver)).toVector + ) + } + JsObject(fields.toMap) + } + + override def addAttachment( + documentId: String, + path: String + ): Either[Throwable, BacklogAttachment] = { + sleep(500) + val file = new File(path) + try { + val attachment = Using.resource(new FileInputStream(file)) { inputStream => + val attachmentData = new AttachmentDataImpl(file.getName, inputStream) + backlog.addDocumentAttachment(documentId, attachmentData) + } + Right( + BacklogAttachment( + optId = Some(attachment.getId), + name = FileUtil.clean(attachment.getName) + ) + ) + } catch { + case e: Throwable => + logger.error(e.getMessage, e) + Left(e) + } + } + + override def addTags( + documentId: String, + tagNames: Seq[String] + ): Either[Throwable, Seq[BacklogDocumentTag]] = + try { + if (tagNames.isEmpty) { + Right(Seq.empty) + } else { + val params = new AddDocumentTagsParams(tagNames.asJava) + val tags = backlog.addDocumentTags(documentId, params).asScala.toSeq + Right(tags.map(tag => BacklogDocumentTag(id = tag.getId, name = tag.getName))) + } + } catch { + case e: Throwable => + logger.error(e.getMessage, e) + Left(e) + } + + private[this] def resolvedUserId( + optUser: Option[BacklogUser], + propertyResolver: PropertyResolver + ): Option[Long] = + for { + user <- optUser + userId <- user.optUserId + id <- propertyResolver.optResolvedUserId(userId) + } yield id + + private[this] def replyFields( + reply: BacklogDocumentCommentReply, + propertyResolver: PropertyResolver + ): JsObject = { + val fields = scala.collection.mutable.Map[String, JsValue]( + "content" -> JsString(reply.content), + "plain" -> JsString(reply.plain) + ) + reply.optCreated.foreach(created => fields += "created" -> JsString(created)) + resolvedUserId(reply.optCreatedUser, propertyResolver) + .foreach(id => fields += "createdUserId" -> JsNumber(id)) + reply.optUpdated.foreach(updated => fields += "updated" -> JsString(updated)) + JsObject(fields.toMap) + } + + override def rewriteInlineCommentIds( + document: BacklogDocument, + commentIdMap: Map[String, String] + ): BacklogDocument = + if (commentIdMap.isEmpty) document + else + document.optJson match { + case Some(json) => + document.copy(optJson = Some(rewriteJsValue(json.parseJson, commentIdMap).compactPrint)) + case None => document + } + + private[this] def rewriteJsValue(value: JsValue, commentIdMap: Map[String, String]): JsValue = + value match { + case JsObject(fields) => + val rewritten = fields.map { case (key, v) => key -> rewriteJsValue(v, commentIdMap) } + rewritten.get("type") match { + case Some(JsString("inlineComment")) => + rewritten.get("attrs") match { + case Some(attrs: JsObject) => + JsObject( + rewritten.updated("attrs", rewriteInlineCommentAttrs(attrs, commentIdMap)) + ) + case _ => JsObject(rewritten) + } + case _ => JsObject(rewritten) + } + case JsArray(elements) => JsArray(elements.map(rewriteJsValue(_, commentIdMap))) + case other => other + } + + private[this] def rewriteInlineCommentAttrs( + attrs: JsObject, + commentIdMap: Map[String, String] + ): JsObject = + attrs.fields.get("comment") match { + case Some(comment: JsObject) => + comment.fields.get("id") match { + case Some(JsString(oldCommentId)) => + commentIdMap.get(oldCommentId) match { + case Some(newCommentId) => + JsObject( + attrs.fields.updated( + "comment", + JsObject(comment.fields.updated("id", JsString(newCommentId))) + ) + ) + case None => + logger.warn( + s"No migrated comment id found for inline comment mark (id=$oldCommentId)" + ) + attrs + } + case _ => attrs + } + case _ => attrs + } + + override def rewriteIssueMentions( + document: BacklogDocument, + issueIdMap: Map[Long, Long], + issueKeyMap: Map[String, String], + srcProjectId: Long, + srcProjectKey: String, + dstProjectId: Long, + dstProjectKey: String + ): (BacklogDocument, IssueMentionRewriteStats) = + if (issueIdMap.isEmpty && issueKeyMap.isEmpty) (document, IssueMentionRewriteStats(0, 0, 0, 0)) + else + document.optJson match { + case Some(json) => + val plainTextReplacements = + scala.collection.mutable.ArrayBuffer.empty[(String, String, String)] + val ctx = IssueMentionContext( + issueIdMap, + issueKeyMap, + srcProjectKey, + dstProjectId, + dstProjectKey, + plainTextReplacements + ) + val newJson = rewriteIssueMentionJsValue(json.parseJson, ctx).compactPrint + val newPlain = + document.optPlain.map(rewritePlainTextIssueMentions(_, plainTextReplacements.toSeq)) + val stats = IssueMentionRewriteStats( + total = ctx.rewrittenCount + ctx.skippedExternalProjectCount + ctx.unresolvedCount, + rewritten = ctx.rewrittenCount, + skippedExternalProject = ctx.skippedExternalProjectCount, + unresolved = ctx.unresolvedCount + ) + (document.copy(optJson = Some(newJson), optPlain = newPlain), stats) + case None => (document, IssueMentionRewriteStats(0, 0, 0, 0)) + } + + private[this] case class IssueMentionContext( + issueIdMap: Map[Long, Long], + issueKeyMap: Map[String, String], + srcProjectKey: String, + dstProjectId: Long, + dstProjectKey: String, + // (oldId, old tag text, new tag text) captured for each resolved mention, applied to optPlain afterwards + plainTextReplacements: scala.collection.mutable.ArrayBuffer[(String, String, String)] + ) { + var rewrittenCount: Int = 0 + var skippedExternalProjectCount: Int = 0 + var unresolvedCount: Int = 0 + } + + private[this] def rewritePlainTextIssueMentions( + plain: String, + replacements: Seq[(String, String, String)] + ): String = + replacements.foldLeft(plain) { + case (text, (oldId, oldTag, newTag)) => + if (text.contains(oldTag)) { + text.replace(oldTag, newTag) + } else { + logger.warn( + s"Could not find expected issue mention text in document plain text (id=$oldId) — plain text left unchanged for this mention" + ) + text + } + } + + private[this] def issueMentionTagText(fields: Map[String, JsValue]): Option[String] = + for { + id <- fields.get("id").collect { case JsString(s) => s } + label <- fields.get("label").collect { case JsString(s) => s } + mentionType <- fields.get("mentionType").collect { case JsString(s) => s } + projectKey <- fields.get("projectKey").collect { case JsString(s) => s } + } yield { + val sb = new StringBuilder("[issueMention id=\"") + .append(id) + .append("\" label=\"") + .append(label) + .append("\" mentionType=\"") + .append(mentionType) + .append("\" projectKey=\"") + .append(projectKey) + .append("\"") + fields.get("projectId").collect { case JsNumber(n) => n }.foreach { n => + sb.append(" projectId=\"").append(n.toString).append("\"") + } + fields.get("issueId").collect { case JsNumber(n) => n }.foreach { n => + sb.append(" issueId=\"").append(n.toString).append("\"") + } + sb.append("]") + sb.toString + } + + private[this] def rewriteIssueMentionJsValue(value: JsValue, ctx: IssueMentionContext): JsValue = + value match { + case JsObject(fields) => + val rewritten = fields.map { case (key, v) => key -> rewriteIssueMentionJsValue(v, ctx) } + rewritten.get("type") match { + case Some(JsString("issueMention")) => + rewritten.get("attrs") match { + case Some(attrs: JsObject) => + JsObject(rewritten.updated("attrs", rewriteIssueMentionAttrs(attrs, ctx))) + case _ => JsObject(rewritten) + } + case _ => JsObject(rewritten) + } + case JsArray(elements) => JsArray(elements.map(rewriteIssueMentionJsValue(_, ctx))) + case other => other + } + + private[this] def rewriteIssueMentionAttrs( + attrs: JsObject, + ctx: IssueMentionContext + ): JsObject = { + val optOldId = attrs.fields.get("id").collect { case JsString(id) => id } + val optProjectKey = attrs.fields.get("projectKey").collect { case JsString(pk) => pk } + + (optOldId, optProjectKey) match { + case (Some(oldId), Some(projectKey)) if projectKey == ctx.srcProjectKey => + resolveAndRewriteIssueMentionAttrs(attrs, oldId, ctx) + case (Some(oldId), Some(projectKey)) => + ctx.skippedExternalProjectCount += 1 + logger.warn( + s"Skipping issue mention for external project (projectKey=$projectKey, id=$oldId) — not part of this migration" + ) + attrs + case _ => attrs + } + } + + private[this] def resolveAndRewriteIssueMentionAttrs( + attrs: JsObject, + oldId: String, + ctx: IssueMentionContext + ): JsObject = { + val optOldIssueId = attrs.fields.get("issueId").collect { case JsNumber(n) => n.toLong } + val optNewIssueId = optOldIssueId.flatMap(ctx.issueIdMap.get) + val optNewKey = ctx.issueKeyMap.get(oldId) + + if (optNewIssueId.isEmpty && optNewKey.isEmpty) { + ctx.unresolvedCount += 1 + logger.warn( + s"No migrated issue found for issue mention (id=$oldId, issueId=$optOldIssueId) — leaving reference unresolved" + ) + attrs + } else { + ctx.rewrittenCount += 1 + var fields = attrs.fields.updated("id", JsString(optNewKey.getOrElse(oldId))) + if (attrs.fields.contains("issueId")) { + fields = fields.updated( + "issueId", + optNewIssueId.map(JsNumber(_)).getOrElse(attrs.fields("issueId")) + ) + } + if (attrs.fields.contains("projectId")) { + fields = fields.updated("projectId", JsNumber(ctx.dstProjectId)) + } + fields = fields.updated("projectKey", JsString(ctx.dstProjectKey)) + + (issueMentionTagText(attrs.fields), issueMentionTagText(fields)) match { + case (Some(oldTag), Some(newTag)) => ctx.plainTextReplacements += ((oldId, oldTag, newTag)) + case _ => () + } + + JsObject(fields) + } + } + private[this] def withComments(document: BacklogDocument): BacklogDocument = { val comments = try { diff --git a/core/src/test/scala/com/nulabinc/backlog/migration/service/DocumentServiceImplSpec.scala b/core/src/test/scala/com/nulabinc/backlog/migration/service/DocumentServiceImplSpec.scala new file mode 100644 index 00000000..96d564de --- /dev/null +++ b/core/src/test/scala/com/nulabinc/backlog/migration/service/DocumentServiceImplSpec.scala @@ -0,0 +1,554 @@ +package com.nulabinc.backlog.migration.service + +import com.google.inject.Guice +import com.nulabinc.backlog.migration.common.conf.BacklogApiConfiguration +import com.nulabinc.backlog.migration.common.domain.{ + BacklogDocument, + BacklogDocumentComment, + BacklogDocumentCommentReply +} +import com.nulabinc.backlog.migration.common.modules.DefaultModule +import com.nulabinc.backlog.migration.common.service.{ + DocumentServiceImpl, + IssueMentionRewriteStats +} +import com.nulabinc.backlog.migration.{SimpleFixture, TestPropertyResolver} +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import spray.json._ + +/** + * @author + * nulab + */ +class DocumentServiceImplSpec extends AnyFlatSpec with Matchers with SimpleFixture { + + def documentService(): DocumentServiceImpl = + Guice + .createInjector( + new DefaultModule(BacklogApiConfiguration("url", "key", "projectKey")) + ) + .getInstance(classOf[DocumentServiceImpl]) + + private val documentCreated = "2015-05-01T16:01:51+09:00" + private val documentUpdated = "2015-05-02T16:01:51+09:00" + + private val document = BacklogDocument( + optId = None, + projectId = projectId, + title = "document title", + optJson = Some("""{"type":"doc","content":[]}"""), + optPlain = Some("plain text"), + optEmoji = Some(":smile:"), + tags = Seq.empty, + attachments = Seq.empty, + comments = Seq.empty, + optCreatedUser = Some(user1), + optCreated = Some(documentCreated), + optUpdatedUser = Some(user2), + optUpdated = Some(documentUpdated) + ) + + "createDocumentJson" should "build the import request body" in { + val propertyResolver = new TestPropertyResolver() + + val json = documentService().createDocumentJson( + projectId, + document, + Some("parentDocumentId"), + addLast = true, + isTrash = false, + propertyResolver + ) + + json.fields("projectId") should be(JsNumber(projectId)) + json.fields("title") should be(JsString("document title")) + json.fields("emoji") should be(JsString(":smile:")) + json.fields("parentId") should be(JsString("parentDocumentId")) + json.fields("addLast") should be(JsBoolean(true)) + json.fields("isTrash") should be(JsBoolean(false)) + json.fields("created") should be(JsString(documentCreated)) + json.fields("createdUserId") should be(JsNumber(userId1)) + json.fields("updated") should be(JsString(documentUpdated)) + json.fields("updatedUserId") should be(JsNumber(userId2)) + json.fields.keySet should not contain "content" + } + + it should "omit optional fields that are not set" in { + val propertyResolver = new TestPropertyResolver() + val minimalDocument = document.copy( + optEmoji = None, + optCreatedUser = None, + optCreated = None, + optUpdatedUser = None, + optUpdated = None + ) + + val json = documentService().createDocumentJson( + projectId, + minimalDocument, + None, + addLast = false, + isTrash = true, + propertyResolver + ) + + json.fields.keySet should contain theSameElementsAs Set( + "projectId", + "title", + "addLast", + "isTrash" + ) + json.fields("isTrash") should be(JsBoolean(true)) + } + + "updateContentJson" should "build the content import request body" in { + val propertyResolver = new TestPropertyResolver() + + val json = documentService().updateContentJson(document, propertyResolver) + + json.fields("json") should be("""{"type":"doc","content":[]}""".parseJson) + json.fields("plain") should be(JsString("plain text")) + json.fields("updated") should be(JsString(documentUpdated)) + json.fields("updatedUserId") should be(JsNumber(userId2)) + json.fields.keySet should not contain "title" + } + + it should "fall back to an empty object and empty string when content is missing" in { + val propertyResolver = new TestPropertyResolver() + val emptyDocument = document.copy(optJson = None, optPlain = None) + + val json = documentService().updateContentJson(emptyDocument, propertyResolver) + + json.fields("json") should be(JsObject.empty) + json.fields("plain") should be(JsString("")) + } + + "commentJson" should "build the comment import request body with replies" in { + val propertyResolver = new TestPropertyResolver() + val reply = BacklogDocumentCommentReply( + optId = None, + content = "reply content", + plain = "reply plain", + optCreatedUser = Some(user3), + optCreated = Some(documentCreated), + optUpdated = None + ) + val comment = BacklogDocumentComment( + optId = None, + statusId = 1, + content = "comment content", + plain = "comment plain", + commentType = "comment", + optCreatedUser = Some(user1), + optCreated = Some(documentCreated), + optUpdated = Some(documentUpdated), + replies = Seq(reply) + ) + + val json = documentService().commentJson(comment, propertyResolver) + + json.fields("content") should be(JsString("comment content")) + json.fields("plain") should be(JsString("comment plain")) + json.fields("statusId") should be(JsNumber(1)) + json.fields("commentType") should be(JsString("comment")) + json.fields("createdUserId") should be(JsNumber(userId1)) + + val replies = json.fields("replies").asInstanceOf[JsArray].elements + replies should have size 1 + val replyJson = replies.head.asJsObject + replyJson.fields("content") should be(JsString("reply content")) + replyJson.fields("plain") should be(JsString("reply plain")) + replyJson.fields("createdUserId") should be(JsNumber(userId3)) + replyJson.fields.keySet should not contain "updatedUserId" + } + + it should "omit the replies field when there are no replies" in { + val propertyResolver = new TestPropertyResolver() + val comment = BacklogDocumentComment( + optId = None, + statusId = 1, + content = "comment content", + plain = "comment plain", + commentType = "comment", + optCreatedUser = None, + optCreated = None, + optUpdated = None, + replies = Seq.empty + ) + + val json = documentService().commentJson(comment, propertyResolver) + + json.fields.keySet should not contain "replies" + } + + "rewriteInlineCommentIds" should "rewrite every inlineComment mark's id using the mapping" in { + val body = + """{"type":"doc","content":[ + |{"type":"paragraph","content":[{"type":"text","text":"test","marks":[ + |{"type":"inlineComment","attrs":{"comment":{"id":"old-1","statusId":0}}} + |]}]}, + |{"type":"paragraph","content":[{"type":"text","text":"foo","marks":[ + |{"type":"inlineComment","attrs":{"comment":{"id":"old-2","statusId":0}}} + |]}]} + |]}""".stripMargin + val documentWithBody = document.copy(optJson = Some(body)) + val commentIdMap = Map("old-1" -> "new-1", "old-2" -> "new-2") + + val rewritten = documentService().rewriteInlineCommentIds(documentWithBody, commentIdMap) + + val marks = rewritten.optJson.get.parseJson.asJsObject + .fields("content") + .asInstanceOf[JsArray] + .elements + .flatMap(_.asJsObject.fields("content").asInstanceOf[JsArray].elements) + .flatMap(_.asJsObject.fields("marks").asInstanceOf[JsArray].elements) + .map(_.asJsObject.fields("attrs").asJsObject.fields("comment").asJsObject.fields("id")) + + marks should contain theSameElementsInOrderAs Seq(JsString("new-1"), JsString("new-2")) + } + + it should "leave the id untouched when no mapping exists for it" in { + val body = + """{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":"test", + |"marks":[{"type":"inlineComment","attrs":{"comment":{"id":"unmapped","statusId":0}}}]}]}]}""".stripMargin + val documentWithBody = document.copy(optJson = Some(body)) + + val rewritten = documentService().rewriteInlineCommentIds(documentWithBody, Map.empty) + + rewritten.optJson should be(Some(body)) + } + + it should "not touch marks other than inlineComment" in { + val body = + """{"type":"doc","content":[{"type":"paragraph","content":[{"type":"text","text":"test", + |"marks":[{"type":"bold"}]}]}]}""".stripMargin + val documentWithBody = document.copy(optJson = Some(body)) + + val rewritten = + documentService().rewriteInlineCommentIds(documentWithBody, Map("old" -> "new")) + + rewritten.optJson.get.parseJson should be(body.parseJson) + } + + "rewriteIssueMentions" should "rewrite a same-project issue mention that has issueId and projectId" in { + val body = + """{"type":"doc","content":[{"type":"issueMention","attrs":{ + |"id":"SRC-85","label":"emoji test","mentionType":"inline", + |"projectKey":"SRC","projectId":100,"issueId":200}}]}""".stripMargin + val documentWithBody = document.copy(optJson = Some(body)) + + val (rewritten, stats) = documentService().rewriteIssueMentions( + documentWithBody, + issueIdMap = Map(200L -> 201L), + issueKeyMap = Map("SRC-85" -> "DST-1"), + srcProjectId = 100L, + srcProjectKey = "SRC", + dstProjectId = 101L, + dstProjectKey = "DST" + ) + + val attrs = rewritten.optJson.get.parseJson.asJsObject + .fields("content") + .asInstanceOf[JsArray] + .elements + .head + .asJsObject + .fields("attrs") + .asJsObject + + attrs.fields("id") should be(JsString("DST-1")) + attrs.fields("issueId") should be(JsNumber(201)) + attrs.fields("projectId") should be(JsNumber(101)) + attrs.fields("projectKey") should be(JsString("DST")) + attrs.fields("label") should be(JsString("emoji test")) + attrs.fields("mentionType") should be(JsString("inline")) + stats should be( + IssueMentionRewriteStats( + total = 1, + rewritten = 1, + skippedExternalProject = 0, + unresolved = 0 + ) + ) + } + + it should "rewrite a same-project issue mention missing issueId/projectId via key-only fallback" in { + val body = + """{"type":"doc","content":[{"type":"issueMention","attrs":{ + |"id":"SRC-84","label":"label missing ids","mentionType":"inline", + |"projectKey":"SRC"}}]}""".stripMargin + val documentWithBody = document.copy(optJson = Some(body)) + + val (rewritten, stats) = documentService().rewriteIssueMentions( + documentWithBody, + issueIdMap = Map.empty, + issueKeyMap = Map("SRC-84" -> "DST-2"), + srcProjectId = 100L, + srcProjectKey = "SRC", + dstProjectId = 101L, + dstProjectKey = "DST" + ) + + val attrs = rewritten.optJson.get.parseJson.asJsObject + .fields("content") + .asInstanceOf[JsArray] + .elements + .head + .asJsObject + .fields("attrs") + .asJsObject + + attrs.fields("id") should be(JsString("DST-2")) + attrs.fields("projectKey") should be(JsString("DST")) + attrs.fields.keySet should not contain "issueId" + attrs.fields.keySet should not contain "projectId" + stats should be( + IssueMentionRewriteStats( + total = 1, + rewritten = 1, + skippedExternalProject = 0, + unresolved = 0 + ) + ) + } + + it should "leave a same-project mention untouched when the issue isn't in either map" in { + val body = + """{"type":"doc","content":[{"type":"issueMention","attrs":{ + |"id":"SRC-99","label":"unmigrated","mentionType":"inline", + |"projectKey":"SRC","projectId":100,"issueId":999}}]}""".stripMargin + val documentWithBody = document.copy(optJson = Some(body)) + + val (rewritten, stats) = documentService().rewriteIssueMentions( + documentWithBody, + issueIdMap = Map(200L -> 201L), + issueKeyMap = Map("SRC-85" -> "DST-1"), + srcProjectId = 100L, + srcProjectKey = "SRC", + dstProjectId = 101L, + dstProjectKey = "DST" + ) + + rewritten.optJson.get.parseJson should be(body.parseJson) + stats should be( + IssueMentionRewriteStats( + total = 1, + rewritten = 0, + skippedExternalProject = 0, + unresolved = 1 + ) + ) + } + + it should "leave a mention pointing at a different project completely untouched" in { + val body = + """{"type":"doc","content":[{"type":"issueMention","attrs":{ + |"id":"OTHER-1","label":"other project issue","mentionType":"inline", + |"projectKey":"OTHER","projectId":300,"issueId":400}}]}""".stripMargin + val documentWithBody = document.copy(optJson = Some(body)) + + val (rewritten, stats) = documentService().rewriteIssueMentions( + documentWithBody, + issueIdMap = Map(400L -> 401L), + issueKeyMap = Map("OTHER-1" -> "DST-9"), + srcProjectId = 100L, + srcProjectKey = "SRC", + dstProjectId = 101L, + dstProjectKey = "DST" + ) + + rewritten.optJson.get.parseJson should be(body.parseJson) + stats should be( + IssueMentionRewriteStats( + total = 1, + rewritten = 0, + skippedExternalProject = 1, + unresolved = 0 + ) + ) + } + + it should "return the document unchanged (no parse round-trip) when both maps are empty" in { + val body = + """{"type":"doc","content":[{"type":"issueMention","attrs":{ + |"id":"SRC-85","label":"emoji test","mentionType":"inline", + |"projectKey":"SRC","projectId":100,"issueId":200}}]}""".stripMargin + val documentWithBody = document.copy(optJson = Some(body)) + + val (rewritten, stats) = documentService().rewriteIssueMentions( + documentWithBody, + issueIdMap = Map.empty, + issueKeyMap = Map.empty, + srcProjectId = 100L, + srcProjectKey = "SRC", + dstProjectId = 101L, + dstProjectKey = "DST" + ) + + rewritten.optJson should be theSameInstanceAs documentWithBody.optJson + stats should be( + IssueMentionRewriteStats( + total = 0, + rewritten = 0, + skippedExternalProject = 0, + unresolved = 0 + ) + ) + } + + it should "rewrite the matching bracket tag in optPlain for a fully-resolved same-project mention" in { + val body = + """{"type":"doc","content":[{"type":"issueMention","attrs":{ + |"id":"SRC-85","label":"emoji test","mentionType":"inline", + |"projectKey":"SRC","projectId":100,"issueId":200}}]}""".stripMargin + val oldTag = + """[issueMention id="SRC-85" label="emoji test" mentionType="inline" projectKey="SRC" projectId="100" issueId="200"]""" + val newTag = + """[issueMention id="DST-1" label="emoji test" mentionType="inline" projectKey="DST" projectId="101" issueId="201"]""" + val documentWithBody = + document.copy(optJson = Some(body), optPlain = Some(s"before $oldTag after")) + + val (rewritten, _) = documentService().rewriteIssueMentions( + documentWithBody, + issueIdMap = Map(200L -> 201L), + issueKeyMap = Map("SRC-85" -> "DST-1"), + srcProjectId = 100L, + srcProjectKey = "SRC", + dstProjectId = 101L, + dstProjectKey = "DST" + ) + + rewritten.optPlain should be(Some(s"before $newTag after")) + } + + it should "rewrite the plain-text tag via key-only fallback without gaining issueId/projectId" in { + val body = + """{"type":"doc","content":[{"type":"issueMention","attrs":{ + |"id":"SRC-84","label":"label missing ids","mentionType":"inline", + |"projectKey":"SRC"}}]}""".stripMargin + val oldTag = + """[issueMention id="SRC-84" label="label missing ids" mentionType="inline" projectKey="SRC"]""" + val newTag = + """[issueMention id="DST-2" label="label missing ids" mentionType="inline" projectKey="DST"]""" + val documentWithBody = + document.copy(optJson = Some(body), optPlain = Some(s"text $oldTag more")) + + val (rewritten, _) = documentService().rewriteIssueMentions( + documentWithBody, + issueIdMap = Map.empty, + issueKeyMap = Map("SRC-84" -> "DST-2"), + srcProjectId = 100L, + srcProjectKey = "SRC", + dstProjectId = 101L, + dstProjectKey = "DST" + ) + + rewritten.optPlain should be(Some(s"text $newTag more")) + } + + it should "leave optPlain unchanged and not throw when the expected old tag text can't be found" in { + val body = + """{"type":"doc","content":[{"type":"issueMention","attrs":{ + |"id":"SRC-85","label":"emoji test","mentionType":"inline", + |"projectKey":"SRC","projectId":100,"issueId":200}}]}""".stripMargin + val drifitngPlain = "this plain text has drifted and no longer contains the mention tag" + val documentWithBody = + document.copy(optJson = Some(body), optPlain = Some(drifitngPlain)) + + val (rewritten, _) = documentService().rewriteIssueMentions( + documentWithBody, + issueIdMap = Map(200L -> 201L), + issueKeyMap = Map("SRC-85" -> "DST-1"), + srcProjectId = 100L, + srcProjectKey = "SRC", + dstProjectId = 101L, + dstProjectKey = "DST" + ) + + rewritten.optPlain should be(Some(drifitngPlain)) + } + + it should "rewrite every issueMention tag in optPlain across label edge cases " + + "(missing ids, brackets, encoded quotes)" in { + val jsonBody = + """{"type":"doc","content":[{"type":"paragraph"},{"type":"paragraph","content":[{"type":"text","text":"text one"}]},{"type":"paragraph","content":[{"type":"issueMention","attrs":{"id":"SRCPROJ-1","label":"label one","mentionType":"inline","projectKey":"SRCPROJ","projectId":1000,"issueId":100001}},{"type":"text","text":" "}]},{"type":"paragraph"},{"type":"paragraph","content":[{"type":"text","text":"text two"}]},{"type":"paragraph","content":[{"type":"issueMention","attrs":{"id":"SRCPROJ-2","label":"label missing ids","mentionType":"inline","projectKey":"SRCPROJ"}},{"type":"text","text":" "}]},{"type":"paragraph"},{"type":"paragraph","content":[{"type":"text","text":"“stray quote“and[escaped]"}]},{"type":"paragraph","content":[{"type":"issueMention","attrs":{"id":"SRCPROJ-3","label":"label with "quote" and [brackets]","mentionType":"inline","projectKey":"SRCPROJ","projectId":1000,"issueId":100003}},{"type":"text","text":" "}]},{"type":"paragraph"}]}""" + + val oldTag1 = + """[issueMention id="SRCPROJ-1" label="label one" mentionType="inline" projectKey="SRCPROJ" projectId="1000" issueId="100001"]""" + val oldTag2 = + """[issueMention id="SRCPROJ-2" label="label missing ids" mentionType="inline" projectKey="SRCPROJ"]""" + val oldTag3 = + """[issueMention id="SRCPROJ-3" label="label with "quote" and [brackets]" mentionType="inline" projectKey="SRCPROJ" projectId="1000" issueId="100003"]""" + + val filler1 = "\n\ntext one\n\n" + val filler2 = " \n\n\n\ntext two\n\n" + val filler3 = " \n\n\n\n“stray quote“and\\[escaped\\]\n\n" + val filler4 = " \n\n" + + val plainBody = + filler1 + oldTag1 + filler2 + oldTag2 + filler3 + oldTag3 + filler4 + + val documentWithBody = document.copy(optJson = Some(jsonBody), optPlain = Some(plainBody)) + + val issueIdMap = Map( + 100001L -> 200001L, + 100003L -> 200003L + ) + val issueKeyMap = Map( + "SRCPROJ-1" -> "DST-1", + "SRCPROJ-2" -> "DST-2", + "SRCPROJ-3" -> "DST-3" + ) + + val (rewritten, _) = documentService().rewriteIssueMentions( + documentWithBody, + issueIdMap = issueIdMap, + issueKeyMap = issueKeyMap, + srcProjectId = 1000L, + srcProjectKey = "SRCPROJ", + dstProjectId = 2000L, + dstProjectKey = "DSTPROJ" + ) + + val newTag1 = + """[issueMention id="DST-1" label="label one" mentionType="inline" projectKey="DSTPROJ" projectId="2000" issueId="200001"]""" + val newTag2 = + """[issueMention id="DST-2" label="label missing ids" mentionType="inline" projectKey="DSTPROJ"]""" + val newTag3 = + """[issueMention id="DST-3" label="label with "quote" and [brackets]" mentionType="inline" projectKey="DSTPROJ" projectId="2000" issueId="200003"]""" + + val expectedPlainBody = + filler1 + newTag1 + filler2 + newTag2 + filler3 + newTag3 + filler4 + + rewritten.optPlain should be(Some(expectedPlainBody)) + + def collectIssueMentionIds(value: JsValue): Seq[String] = value match { + case obj: JsObject => + val here = obj.fields.get("type") match { + case Some(JsString("issueMention")) => + obj.fields + .get("attrs") + .toSeq + .collect { + case attrs: JsObject => + attrs.fields("id") + } + .collect { case JsString(id) => id } + case _ => Seq.empty + } + here ++ obj.fields.values.flatMap(collectIssueMentionIds) + case JsArray(elements) => elements.flatMap(collectIssueMentionIds) + case _ => Seq.empty + } + + collectIssueMentionIds( + rewritten.optJson.get.parseJson + ) should contain theSameElementsInOrderAs Seq( + "DST-1", + "DST-2", + "DST-3" + ) + } + +} diff --git a/importer/src/main/scala/com/nulabinc/backlog/migration/importer/service/DocumentsImporter.scala b/importer/src/main/scala/com/nulabinc/backlog/migration/importer/service/DocumentsImporter.scala new file mode 100644 index 00000000..1ecb3bdf --- /dev/null +++ b/importer/src/main/scala/com/nulabinc/backlog/migration/importer/service/DocumentsImporter.scala @@ -0,0 +1,264 @@ +package com.nulabinc.backlog.migration.importer.service + +import javax.inject.Inject + +import better.files.{File => Path} +import com.nulabinc.backlog.migration.common.conf.BacklogPaths +import com.nulabinc.backlog.migration.common.convert.BacklogUnmarshaller +import com.nulabinc.backlog.migration.common.domain.{ + BacklogAttachment, + BacklogDocument, + BacklogDocumentTreeNode, + BacklogProject +} +import com.nulabinc.backlog.migration.common.dsl.ConsoleDSL +import com.nulabinc.backlog.migration.common.service.{ + DocumentService, + IssueMentionRewriteStats, + PropertyResolver +} +import com.nulabinc.backlog.migration.common.utils.Logging +import com.osinka.i18n.Messages +import monix.eval.Task +import monix.execution.Scheduler +import org.fusesource.jansi.Ansi.Color.GREEN + +/** + * @author + * nulab + */ +private[importer] class DocumentsImporter @Inject() ( + backlogPaths: BacklogPaths, + documentService: DocumentService +) extends Logging { + + def execute( + project: BacklogProject, + propertyResolver: PropertyResolver, + issueIdMap: Map[Long, Long], + issueKeyMap: Map[String, String], + srcProjectId: Long, + srcProjectKey: String + )(implicit + s: Scheduler, + consoleDSL: ConsoleDSL[Task] + ): Unit = + BacklogUnmarshaller.documentTree(backlogPaths).foreach { tree => + val issueMentionContext = IssueMentionContext( + issueIdMap, + issueKeyMap, + srcProjectId, + srcProjectKey, + dstProjectId = project.id, + dstProjectKey = project.key + ) + + walk( + tree.activeTree.children, + None, + isTrash = false, + project, + propertyResolver, + issueMentionContext + ) + walk( + tree.trashTree.children, + None, + isTrash = true, + project, + propertyResolver, + issueMentionContext + ) + } + + private[this] case class IssueMentionContext( + issueIdMap: Map[Long, Long], + issueKeyMap: Map[String, String], + srcProjectId: Long, + srcProjectKey: String, + dstProjectId: Long, + dstProjectKey: String + ) + + private[this] def walk( + nodes: Seq[BacklogDocumentTreeNode], + optNewParentId: Option[String], + isTrash: Boolean, + project: BacklogProject, + propertyResolver: PropertyResolver, + issueMentionContext: IssueMentionContext + )(implicit s: Scheduler, consoleDSL: ConsoleDSL[Task]): Unit = + nodes.foreach { node => + val optNewId = unmarshal(node.id).map { document => + // isTrash is only consulted by the destination when optNewParentId is + // empty (root of the subtree); it's harmless to pass through unconditionally. + val newId = documentService.create( + project.id, + document, + optNewParentId, + addLast = true, + isTrash = isTrash, + propertyResolver + ) + postCreate(node.id, newId, document, propertyResolver, issueMentionContext).runSyncUnsafe() + newId + } + // A failed/missing parent breaks the id mapping, so its children are skipped too. + optNewId.foreach { newId => + walk(node.children, Some(newId), isTrash, project, propertyResolver, issueMentionContext) + } + } + + private[this] def postCreate( + oldDocumentId: String, + newDocumentId: String, + document: BacklogDocument, + propertyResolver: PropertyResolver, + issueMentionContext: IssueMentionContext + )(implicit consoleDSL: ConsoleDSL[Task]): Task[Unit] = + for { + _ <- ConsoleDSL[Task].println(s"[Document id=$newDocumentId]") + _ <- logStep("Document created(title, emoji)", ok = true) + // Comments must be created before the content update: the body's + // inlineComment marks reference comment ids, which only exist once + // comments have been (re-)created at the destination. + commentIdMap <- postComments(newDocumentId, document, propertyResolver) + _ <- logStepCount("Comments imported", commentIdMap.size, document.comments.size) + rewriteResult <- Task { + val withRewrittenComments = + documentService.rewriteInlineCommentIds(document, commentIdMap) + documentService.rewriteIssueMentions( + withRewrittenComments, + issueMentionContext.issueIdMap, + issueMentionContext.issueKeyMap, + issueMentionContext.srcProjectId, + issueMentionContext.srcProjectKey, + issueMentionContext.dstProjectId, + issueMentionContext.dstProjectKey + ) + } + _ <- logStep("Document content rewritten", ok = true) + _ <- logIssueMentionStep(rewriteResult._2) + _ <- Task(documentService.updateContent(newDocumentId, rewriteResult._1, propertyResolver)) + _ <- logStep("Document content updated", ok = true) + tagsResult <- postTags(newDocumentId, document) + _ <- logStepCount("Tags added", tagsResult._1, tagsResult._2) + attachmentsResult <- postAttachments(oldDocumentId, newDocumentId, document) + _ <- logStepCount( + "Attachments added", + attachmentsResult._1, + attachmentsResult._2 + ) + } yield () + + // Always OK: reaching this call means the step didn't throw. + private[this] def logStep(label: String, ok: Boolean)(implicit + consoleDSL: ConsoleDSL[Task] + ): Task[Unit] = + if (ok) ConsoleDSL[Task].println(s"$label: OK", space = 2, color = GREEN) + else ConsoleDSL[Task].errorln(s"$label: NG", space = 2) + + // e.g. "Comments imported: OK (3)" or "Attachments added: NG (1/2)". + private[this] def logStepCount(label: String, success: Int, total: Int)(implicit + consoleDSL: ConsoleDSL[Task] + ): Task[Unit] = + if (success == total) + ConsoleDSL[Task].println(s"$label: OK ($total)", space = 2, color = GREEN) + else ConsoleDSL[Task].errorln(s"$label: NG ($success/$total)", space = 2) + + // Skipped mentions aren't failures; only unresolved ones make this NG. + private[this] def logIssueMentionStep(stats: IssueMentionRewriteStats)(implicit + consoleDSL: ConsoleDSL[Task] + ): Task[Unit] = { + val details = Seq( + Option.when(stats.skippedExternalProject > 0)(s"${stats.skippedExternalProject} skipped"), + Option.when(stats.unresolved > 0)(s"${stats.unresolved} unresolved") + ).flatten + val suffix = if (details.isEmpty) "" else s", ${details.mkString(", ")}" + val countText = s"${stats.rewritten}/${stats.total}$suffix" + if (stats.unresolved == 0) + ConsoleDSL[Task].println( + s"Issue mentions rewritten: OK ($countText)", + space = 2, + color = GREEN + ) + else + ConsoleDSL[Task].errorln(s"Issue mentions rewritten: NG ($countText)", space = 2) + } + + private[this] def postAttachments( + oldDocumentId: String, + newDocumentId: String, + document: BacklogDocument + )(implicit consoleDSL: ConsoleDSL[Task]): Task[(Int, Int)] = { + val total = document.attachments.size + Task + .sequence(document.attachments.map { attachment => + toPath(oldDocumentId, attachment) match { + case Some(path) => + documentService.addAttachment(newDocumentId, path.pathAsString) match { + case Right(_) => Task(true) + case Left(e) => + ConsoleDSL[Task] + .errorln( + Messages("import.error.document.attachment", attachment.name, e.getMessage) + ) + .map(_ => false) + } + case None => + logger.warn(s"${attachment.name} does not exist") + Task(false) + } + }) + .map(results => (results.count(identity), total)) + } + + private[this] def toPath(oldDocumentId: String, attachment: BacklogAttachment): Option[Path] = + attachment.optId + .map(id => backlogPaths.documentAttachmentPath(oldDocumentId, s"${id}_${attachment.name}")) + .filter(_.exists) + + private[this] def postTags( + newDocumentId: String, + document: BacklogDocument + )(implicit consoleDSL: ConsoleDSL[Task]): Task[(Int, Int)] = { + val tagNames = document.tags.map(_.name) + val total = tagNames.size + if (tagNames.isEmpty) Task((0, 0)) + else + documentService.addTags(newDocumentId, tagNames) match { + case Right(_) => Task((total, total)) + case Left(e) => + ConsoleDSL[Task] + .errorln(Messages("import.error.document.tags", document.title, e.getMessage)) + .map(_ => (0, total)) + } + } + + // Returns a map of old (source-space) comment id -> new (destination-space) + // comment id, so the document body's inlineComment marks can be rewritten + // to point at the newly created comments. + private[this] def postComments( + newDocumentId: String, + document: BacklogDocument, + propertyResolver: PropertyResolver + )(implicit consoleDSL: ConsoleDSL[Task]): Task[Map[String, String]] = + Task + .sequence(document.comments.map { comment => + documentService.addComment(newDocumentId, comment, propertyResolver) match { + case Right(newCommentId) => + Task(comment.optId.map(oldCommentId => oldCommentId -> newCommentId)) + case Left(e) => + ConsoleDSL[Task] + .errorln( + Messages("import.error.document.comment", document.title, e.getMessage) + ) + .map(_ => None) + } + }) + .map(_.flatten.toMap) + + private[this] def unmarshal(documentId: String): Option[BacklogDocument] = + BacklogUnmarshaller.document(backlogPaths.documentJson(documentId)) + +} diff --git a/importer/src/main/scala/com/nulabinc/backlog/migration/importer/service/IssueContext.scala b/importer/src/main/scala/com/nulabinc/backlog/migration/importer/service/IssueContext.scala index 0fd12ae2..cafd09db 100644 --- a/importer/src/main/scala/com/nulabinc/backlog/migration/importer/service/IssueContext.scala +++ b/importer/src/main/scala/com/nulabinc/backlog/migration/importer/service/IssueContext.scala @@ -19,10 +19,15 @@ private[importer] case class IssueContext( val toRemoteIssueId = (localIssueId: Long) => issueIdMap.get(localIssueId): Option[Long] val excludeIssueIds: mutable.ArrayBuffer[Long] = mutable.ArrayBuffer() - private[this] val issueIdMap: mutable.Map[Long, Long] = mutable.Map() + private[this] val issueIdMap: mutable.Map[Long, Long] = mutable.Map() + private[this] val issueKeyMap: mutable.Map[String, String] = mutable.Map() def addIssueId(backlogIssue: BacklogIssue, remoteIssue: BacklogIssue) = { - issueIdMap += backlogIssue.id -> remoteIssue.id + issueIdMap += backlogIssue.id -> remoteIssue.id + issueKeyMap += backlogIssue.issueKey -> remoteIssue.issueKey } + def issueIdMapSnapshot: Map[Long, Long] = issueIdMap.toMap + def issueKeyMapSnapshot: Map[String, String] = issueKeyMap.toMap + } diff --git a/importer/src/main/scala/com/nulabinc/backlog/migration/importer/service/IssueProgressBar.scala b/importer/src/main/scala/com/nulabinc/backlog/migration/importer/service/IssueProgressBar.scala index 0a05aee5..389ef214 100644 --- a/importer/src/main/scala/com/nulabinc/backlog/migration/importer/service/IssueProgressBar.scala +++ b/importer/src/main/scala/com/nulabinc/backlog/migration/importer/service/IssueProgressBar.scala @@ -87,6 +87,18 @@ private[importer] class IssueProgressBar() extends Logging { newLine = false } + // Removes the last call's extra separator/remaining-time lines, leaving + // just the summary. Callers must call this when done — nothing else knows + // how many filler lines to clean up. + def finish(): Unit = { + (0 until 2).foreach { _ => + ConsoleOut.outStream.print( + ansi.cursorLeft(999).cursorUp(1).eraseLine(Ansi.Erase.ALL) + ) + } + ConsoleOut.outStream.flush() + } + private[this] def current(indexOfDate: Int, totalOfDate: Int): String = { val progressBar = ProgressBar.progressBar(indexOfDate, totalOfDate) val resultString = diff --git a/importer/src/main/scala/com/nulabinc/backlog/migration/importer/service/IssuesImporter.scala b/importer/src/main/scala/com/nulabinc/backlog/migration/importer/service/IssuesImporter.scala index 7e8cf554..ce371276 100644 --- a/importer/src/main/scala/com/nulabinc/backlog/migration/importer/service/IssuesImporter.scala +++ b/importer/src/main/scala/com/nulabinc/backlog/migration/importer/service/IssuesImporter.scala @@ -41,7 +41,11 @@ private[importer] class IssuesImporter( propertyResolver: PropertyResolver, fitIssueKey: Boolean, retryCount: Int - )(implicit s: Scheduler, storeDSL: StoreDSL[Task], consoleDSL: ConsoleDSL[Task]): Task[Unit] = { + )(implicit + s: Scheduler, + storeDSL: StoreDSL[Task], + consoleDSL: ConsoleDSL[Task] + ): Task[(Map[Long, Long], Map[String, String])] = { for { _ <- ConsoleDSL[Task].println(""" @@ -55,6 +59,10 @@ private[importer] class IssuesImporter( paths.foreach { path => loadDateDirectory(project, path) } + // Clean up now — nothing downstream is guaranteed to run right after + // this importer (e.g. document import may follow). + if (console.totalSize > 0) console.finish() + (context.issueIdMapSnapshot, context.issueKeyMapSnapshot) } } diff --git a/importer/src/main/scala/com/nulabinc/backlog/migration/importer/service/ProjectImporter.scala b/importer/src/main/scala/com/nulabinc/backlog/migration/importer/service/ProjectImporter.scala index 4f816faa..38eb4400 100644 --- a/importer/src/main/scala/com/nulabinc/backlog/migration/importer/service/ProjectImporter.scala +++ b/importer/src/main/scala/com/nulabinc/backlog/migration/importer/service/ProjectImporter.scala @@ -17,8 +17,6 @@ import com.nulabinc.backlog4j.BacklogAPIException import com.osinka.i18n.Messages import monix.eval.Task import monix.execution.Scheduler -import org.fusesource.jansi.Ansi -import org.fusesource.jansi.Ansi.ansi import scala.util.Try @@ -39,6 +37,7 @@ private[importer] class ProjectImporter @Inject() ( issueCategoryService: IssueCategoryService, customFieldSettingService: CustomFieldSettingService, wikisImporter: WikisImporter, + documentsImporter: DocumentsImporter, resolutionService: ResolutionService, userService: UserService, sharedFileService: SharedFileService, @@ -50,20 +49,13 @@ private[importer] class ProjectImporter @Inject() ( fitIssueKey: Boolean, retryCount: Int )(implicit s: Scheduler, storeDSL: StoreDSL[Task], consoleDSL: ConsoleDSL[Task]): Task[Unit] = { - val project = BacklogUnmarshaller.project(backlogPaths) - projectService.create(project) match { + val srcProject = BacklogUnmarshaller.project(backlogPaths) + projectService.create(srcProject) match { case Right(project) => for { _ <- preExecute() - _ <- contents(project, fitIssueKey, retryCount) + _ <- contents(project, srcProject.id, srcProject.key, fitIssueKey, retryCount) _ <- postExecute() - _ <- ConsoleDSL[Task].printStream( - ansi.cursorLeft(999).cursorUp(1).eraseLine(Ansi.Erase.ALL) - ) - _ <- ConsoleDSL[Task].printStream( - ansi.cursorLeft(999).cursorUp(1).eraseLine(Ansi.Erase.ALL) - ) - _ <- ConsoleDSL[Task].flush() _ <- ConsoleDSL[Task].println(ConsoleMessages.Imports.finish) } yield () case Left(e) => @@ -71,12 +63,12 @@ private[importer] class ProjectImporter @Inject() ( val message = if (e.getMessage.contains("Project limit.")) - Errors.limitProject(project.key) + Errors.limitProject(srcProject.key) else if (e.getMessage.contains("Duplicate entry")) - Errors.projectNotJoin(project.key) + Errors.projectNotJoin(srcProject.key) else { logger.error(e.getMessage, e) - Errors.failed(project.key, e.getMessage()) + Errors.failed(srcProject.key, e.getMessage()) } for { _ <- ConsoleDSL[Task].errorln(message) @@ -87,6 +79,8 @@ private[importer] class ProjectImporter @Inject() ( private def contents( project: BacklogProject, + srcProjectId: Long, + srcProjectKey: String, fitIssueKey: Boolean, retryCount: Int )(implicit s: Scheduler, storeDSL: StoreDSL[Task], consoleDSL: ConsoleDSL[Task]): Task[Unit] = { @@ -105,7 +99,22 @@ private[importer] class ProjectImporter @Inject() ( } // Issue - issuesImporter.execute(project, propertyResolver, fitIssueKey, retryCount) + issuesImporter + .execute(project, propertyResolver, fitIssueKey, retryCount) + .map { + case (issueIdMap, issueKeyMap) => + if (project.useDocument) { + // Document + documentsImporter.execute( + project, + propertyResolver, + issueIdMap, + issueKeyMap, + srcProjectId, + srcProjectKey + ) + } + } } private def preExecute()(implicit