Roman Elizarov | f16fd27 | 2017-02-07 11:26:00 +0300 | [diff] [blame] | 1 | /* |
Roman Elizarov | 1f74a2d | 2018-06-29 19:19:45 +0300 | [diff] [blame^] | 2 | * Copyright 2016-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. |
Roman Elizarov | f16fd27 | 2017-02-07 11:26:00 +0300 | [diff] [blame] | 3 | */ |
| 4 | |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 5 | import java.io.File |
| 6 | import java.io.IOException |
| 7 | import java.io.LineNumberReader |
| 8 | import java.io.Reader |
| 9 | import java.util.* |
| 10 | import kotlin.properties.Delegates |
| 11 | |
| 12 | // --- props in knit.properties |
| 13 | |
| 14 | val knitProperties = ClassLoader.getSystemClassLoader() |
| 15 | .getResource("knit.properties").openStream().use { Properties().apply { load(it) } } |
| 16 | |
| 17 | val siteRoot = knitProperties.getProperty("site.root")!! |
| 18 | val moduleRoots = knitProperties.getProperty("module.roots").split(" ") |
| 19 | val moduleMarker = knitProperties.getProperty("module.marker")!! |
| 20 | val moduleDocs = knitProperties.getProperty("module.docs")!! |
| 21 | |
| 22 | // --- markdown syntax |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 23 | |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 24 | const val DIRECTIVE_START = "<!--- " |
| 25 | const val DIRECTIVE_END = "-->" |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 26 | |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 27 | const val TOC_DIRECTIVE = "TOC" |
| 28 | const val KNIT_DIRECTIVE = "KNIT" |
| 29 | const val INCLUDE_DIRECTIVE = "INCLUDE" |
| 30 | const val CLEAR_DIRECTIVE = "CLEAR" |
| 31 | const val TEST_DIRECTIVE = "TEST" |
Roman Elizarov | e0c817d | 2017-02-10 10:22:01 +0300 | [diff] [blame] | 32 | |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 33 | const val TEST_OUT_DIRECTIVE = "TEST_OUT" |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 34 | |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 35 | const val MODULE_DIRECTIVE = "MODULE" |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 36 | const val INDEX_DIRECTIVE = "INDEX" |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 37 | |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 38 | const val CODE_START = "```kotlin" |
| 39 | const val CODE_END = "```" |
| 40 | |
| 41 | const val TEST_START = "```text" |
| 42 | const val TEST_END = "```" |
| 43 | |
| 44 | const val SECTION_START = "##" |
| 45 | |
| 46 | const val PACKAGE_PREFIX = "package " |
| 47 | const val STARTS_WITH_PREDICATE = "STARTS_WITH" |
Roman Elizarov | 1e45960 | 2017-02-27 11:05:17 +0300 | [diff] [blame] | 48 | const val ARBITRARY_TIME_PREDICATE = "ARBITRARY_TIME" |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 49 | const val FLEXIBLE_TIME_PREDICATE = "FLEXIBLE_TIME" |
| 50 | const val FLEXIBLE_THREAD_PREDICATE = "FLEXIBLE_THREAD" |
| 51 | const val LINES_START_UNORDERED_PREDICATE = "LINES_START_UNORDERED" |
| 52 | const val LINES_START_PREDICATE = "LINES_START" |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 53 | |
Roman Elizarov | 8839673 | 2017-09-27 21:30:47 +0300 | [diff] [blame] | 54 | val API_REF_REGEX = Regex("(^|[ \\]])\\[([A-Za-z0-9_().]+)\\]($|[^\\[\\(])") |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 55 | |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 56 | fun main(args: Array<String>) { |
Roman Elizarov | a5e653f | 2017-02-13 13:49:55 +0300 | [diff] [blame] | 57 | if (args.isEmpty()) { |
| 58 | println("Usage: Knit <markdown-files>") |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 59 | return |
| 60 | } |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 61 | args.forEach { |
| 62 | if (!knit(it)) System.exit(1) // abort on first error with error exit code |
| 63 | } |
Roman Elizarov | a5e653f | 2017-02-13 13:49:55 +0300 | [diff] [blame] | 64 | } |
| 65 | |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 66 | fun knit(markdownFileName: String): Boolean { |
Roman Elizarov | a5e653f | 2017-02-13 13:49:55 +0300 | [diff] [blame] | 67 | println("*** Reading $markdownFileName") |
| 68 | val markdownFile = File(markdownFileName) |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 69 | val tocLines = arrayListOf<String>() |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 70 | var knitRegex: Regex? = null |
| 71 | val includes = arrayListOf<Include>() |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 72 | val codeLines = arrayListOf<String>() |
| 73 | val testLines = arrayListOf<String>() |
| 74 | var testOut: String? = null |
| 75 | val testOutLines = arrayListOf<String>() |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 76 | var lastPgk: String? = null |
Roman Elizarov | 23f864e | 2017-03-03 19:57:47 +0300 | [diff] [blame] | 77 | val files = mutableSetOf<File>() |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 78 | val allApiRefs = arrayListOf<ApiRef>() |
| 79 | val remainingApiRefNames = mutableSetOf<String>() |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 80 | var moduleName: String by Delegates.notNull() |
| 81 | var docsRoot: String by Delegates.notNull() |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 82 | // read markdown file |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 83 | var putBackLine: String? = null |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 84 | val markdown = markdownFile.withMarkdownTextReader { |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 85 | mainLoop@ while (true) { |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 86 | val inLine = putBackLine ?: readLine() ?: break |
| 87 | putBackLine = null |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 88 | val directive = directive(inLine) |
| 89 | if (directive != null && markdownPart == MarkdownPart.TOC) { |
| 90 | markdownPart = MarkdownPart.POST_TOC |
| 91 | postTocText += inLine |
| 92 | } |
| 93 | when (directive?.name) { |
| 94 | TOC_DIRECTIVE -> { |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 95 | requireSingleLine(directive) |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 96 | require(directive.param.isEmpty()) { "$TOC_DIRECTIVE directive must not have parameters" } |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 97 | require(markdownPart == MarkdownPart.PRE_TOC) { "Only one TOC directive is supported" } |
| 98 | markdownPart = MarkdownPart.TOC |
| 99 | } |
| 100 | KNIT_DIRECTIVE -> { |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 101 | requireSingleLine(directive) |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 102 | require(!directive.param.isEmpty()) { "$KNIT_DIRECTIVE directive must include regex parameter" } |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 103 | require(knitRegex == null) { "Only one KNIT directive is supported"} |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 104 | knitRegex = Regex("\\((" + directive.param + ")\\)") |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 105 | continue@mainLoop |
| 106 | } |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 107 | INCLUDE_DIRECTIVE -> { |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 108 | if (directive.param.isEmpty()) { |
| 109 | require(!directive.singleLine) { "$INCLUDE_DIRECTIVE directive without parameters must not be single line" } |
| 110 | readUntilTo(DIRECTIVE_END, codeLines) |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 111 | } else { |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 112 | val include = Include(Regex(directive.param)) |
| 113 | if (directive.singleLine) { |
| 114 | include.lines += codeLines |
| 115 | codeLines.clear() |
| 116 | } else { |
| 117 | readUntilTo(DIRECTIVE_END, include.lines) |
| 118 | } |
| 119 | includes += include |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 120 | } |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 121 | continue@mainLoop |
| 122 | } |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 123 | CLEAR_DIRECTIVE -> { |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 124 | requireSingleLine(directive) |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 125 | require(directive.param.isEmpty()) { "$CLEAR_DIRECTIVE directive must not have parameters" } |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 126 | codeLines.clear() |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 127 | continue@mainLoop |
| 128 | } |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 129 | TEST_OUT_DIRECTIVE -> { |
| 130 | require(!directive.param.isEmpty()) { "$TEST_OUT_DIRECTIVE directive must include file name parameter" } |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 131 | flushTestOut(markdownFile.parentFile, testOut, testOutLines) |
| 132 | testOut = directive.param |
| 133 | readUntil(DIRECTIVE_END).forEach { testOutLines += it } |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 134 | } |
| 135 | TEST_DIRECTIVE -> { |
| 136 | require(lastPgk != null) { "'$PACKAGE_PREFIX' prefix was not found in emitted code"} |
| 137 | require(testOut != null) { "$TEST_OUT_DIRECTIVE directive was not specified" } |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 138 | val predicate = directive.param |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 139 | if (testLines.isEmpty()) { |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 140 | if (directive.singleLine) { |
| 141 | require(!predicate.isEmpty()) { "$TEST_OUT_DIRECTIVE must be preceded by $TEST_START block or contain test predicate"} |
| 142 | } else |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 143 | testLines += readUntil(DIRECTIVE_END) |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 144 | } else { |
| 145 | requireSingleLine(directive) |
| 146 | } |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 147 | makeTest(testOutLines, lastPgk!!, testLines, predicate) |
| 148 | testLines.clear() |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 149 | } |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 150 | MODULE_DIRECTIVE -> { |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 151 | requireSingleLine(directive) |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 152 | moduleName = directive.param |
| 153 | docsRoot = findModuleRootDir(moduleName) + "/" + moduleDocs + "/" + moduleName |
Roman Elizarov | e0c817d | 2017-02-10 10:22:01 +0300 | [diff] [blame] | 154 | } |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 155 | INDEX_DIRECTIVE -> { |
| 156 | requireSingleLine(directive) |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 157 | val indexLines = processApiIndex(siteRoot + "/" + moduleName, docsRoot, directive.param, remainingApiRefNames) |
| 158 | ?: throw IllegalArgumentException("Failed to load index for ${directive.param}") |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 159 | skip = true |
| 160 | while (true) { |
| 161 | val skipLine = readLine() ?: break@mainLoop |
| 162 | if (directive(skipLine) != null) { |
| 163 | putBackLine = skipLine |
| 164 | break |
| 165 | } |
| 166 | } |
| 167 | skip = false |
Roman Elizarov | a5e653f | 2017-02-13 13:49:55 +0300 | [diff] [blame] | 168 | outText += indexLines |
| 169 | outText += putBackLine!! |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 170 | } |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 171 | } |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 172 | if (inLine.startsWith(CODE_START)) { |
Roman Elizarov | f724f6e | 2017-04-07 18:06:22 +0300 | [diff] [blame] | 173 | require(testOut == null || testLines.isEmpty()) { "Previous test was not emitted with $TEST_DIRECTIVE" } |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 174 | codeLines += "" |
| 175 | readUntilTo(CODE_END, codeLines) |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 176 | continue@mainLoop |
| 177 | } |
| 178 | if (inLine.startsWith(TEST_START)) { |
Roman Elizarov | f724f6e | 2017-04-07 18:06:22 +0300 | [diff] [blame] | 179 | require(testOut == null || testLines.isEmpty()) { "Previous test was not emitted with $TEST_DIRECTIVE" } |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 180 | readUntilTo(TEST_END, testLines) |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 181 | continue@mainLoop |
| 182 | } |
| 183 | if (inLine.startsWith(SECTION_START) && markdownPart == MarkdownPart.POST_TOC) { |
| 184 | val i = inLine.indexOf(' ') |
| 185 | require(i >= 2) { "Invalid section start" } |
| 186 | val name = inLine.substring(i + 1).trim() |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 187 | tocLines += " ".repeat(i - 2) + "* [$name](#${makeSectionRef(name)})" |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 188 | continue@mainLoop |
| 189 | } |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 190 | for (match in API_REF_REGEX.findAll(inLine)) { |
| 191 | val apiRef = ApiRef(lineNumber, match.groups[2]!!.value) |
| 192 | allApiRefs += apiRef |
| 193 | remainingApiRefNames += apiRef.name |
| 194 | } |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 195 | knitRegex?.find(inLine)?.let { knitMatch -> |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 196 | val fileName = knitMatch.groups[1]!!.value |
Roman Elizarov | 23f864e | 2017-03-03 19:57:47 +0300 | [diff] [blame] | 197 | val file = File(markdownFile.parentFile, fileName) |
| 198 | require(files.add(file)) { "Duplicate file: $file"} |
| 199 | println("Knitting $file ...") |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 200 | val outLines = arrayListOf<String>() |
| 201 | for (include in includes) { |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 202 | val includeMatch = include.regex.matchEntire(fileName) ?: continue |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 203 | include.lines.forEach { includeLine -> |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 204 | val line = makeReplacements(includeLine, includeMatch) |
| 205 | if (line.startsWith(PACKAGE_PREFIX)) |
| 206 | lastPgk = line.substring(PACKAGE_PREFIX.length).trim() |
| 207 | outLines += line |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 208 | } |
| 209 | } |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 210 | outLines += codeLines |
| 211 | codeLines.clear() |
| 212 | writeLinesIfNeeded(file, outLines) |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 213 | } |
| 214 | } |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 215 | } ?: return false // false when failed |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 216 | // update markdown file with toc |
Roman Elizarov | a5e653f | 2017-02-13 13:49:55 +0300 | [diff] [blame] | 217 | val newLines = buildList<String> { |
| 218 | addAll(markdown.preTocText) |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 219 | if (!tocLines.isEmpty()) { |
Roman Elizarov | a5e653f | 2017-02-13 13:49:55 +0300 | [diff] [blame] | 220 | add("") |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 221 | addAll(tocLines) |
Roman Elizarov | a5e653f | 2017-02-13 13:49:55 +0300 | [diff] [blame] | 222 | add("") |
| 223 | } |
| 224 | addAll(markdown.postTocText) |
| 225 | } |
| 226 | if (newLines != markdown.inText) writeLines(markdownFile, newLines) |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 227 | // check apiRefs |
| 228 | for (apiRef in allApiRefs) { |
| 229 | if (apiRef.name in remainingApiRefNames) { |
| 230 | println("WARNING: $markdownFile: ${apiRef.line}: Broken reference to [${apiRef.name}]") |
| 231 | } |
| 232 | } |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 233 | // write test output |
| 234 | flushTestOut(markdownFile.parentFile, testOut, testOutLines) |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 235 | return true |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 236 | } |
| 237 | |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 238 | fun makeTest(testOutLines: MutableList<String>, pgk: String, test: List<String>, predicate: String) { |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 239 | val funName = buildString { |
| 240 | var cap = true |
| 241 | for (c in pgk) { |
| 242 | if (c == '.') { |
| 243 | cap = true |
| 244 | } else { |
| 245 | append(if (cap) c.toUpperCase() else c) |
| 246 | cap = false |
| 247 | } |
| 248 | } |
| 249 | } |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 250 | testOutLines += "" |
| 251 | testOutLines += " @Test" |
| 252 | testOutLines += " fun test$funName() {" |
Roman Elizarov | ba0c004 | 2017-07-12 13:01:17 +0300 | [diff] [blame] | 253 | val prefix = " test(\"$funName\") { $pgk.main(emptyArray()) }" |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 254 | when (predicate) { |
| 255 | "" -> makeTestLines(testOutLines, prefix, "verifyLines", test) |
| 256 | STARTS_WITH_PREDICATE -> makeTestLines(testOutLines, prefix, "verifyLinesStartWith", test) |
| 257 | ARBITRARY_TIME_PREDICATE -> makeTestLines(testOutLines, prefix, "verifyLinesArbitraryTime", test) |
| 258 | FLEXIBLE_TIME_PREDICATE -> makeTestLines(testOutLines, prefix, "verifyLinesFlexibleTime", test) |
| 259 | FLEXIBLE_THREAD_PREDICATE -> makeTestLines(testOutLines, prefix, "verifyLinesFlexibleThread", test) |
| 260 | LINES_START_UNORDERED_PREDICATE -> makeTestLines(testOutLines, prefix, "verifyLinesStartUnordered", test) |
| 261 | LINES_START_PREDICATE -> makeTestLines(testOutLines, prefix, "verifyLinesStart", test) |
| 262 | else -> { |
| 263 | testOutLines += prefix + ".also { lines ->" |
| 264 | testOutLines += " check($predicate)" |
| 265 | testOutLines += " }" |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 266 | } |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 267 | } |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 268 | testOutLines += " }" |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 269 | } |
| 270 | |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 271 | private fun makeTestLines(testOutLines: MutableList<String>, prefix: String, method: String, test: List<String>) { |
| 272 | testOutLines += "$prefix.$method(" |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 273 | for ((index, testLine) in test.withIndex()) { |
| 274 | val commaOpt = if (index < test.size - 1) "," else "" |
| 275 | val escapedLine = testLine.replace("\"", "\\\"") |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 276 | testOutLines += " \"$escapedLine\"$commaOpt" |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 277 | } |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 278 | testOutLines += " )" |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 279 | } |
| 280 | |
| 281 | private fun makeReplacements(line: String, match: MatchResult): String { |
| 282 | var result = line |
| 283 | for ((id, group) in match.groups.withIndex()) { |
| 284 | if (group != null) |
| 285 | result = result.replace("\$\$$id", group.value) |
| 286 | } |
| 287 | return result |
| 288 | } |
| 289 | |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 290 | private fun flushTestOut(parentDir: File?, testOut: String?, testOutLines: MutableList<String>) { |
| 291 | if (testOut == null) return |
| 292 | val file = File(parentDir, testOut) |
| 293 | testOutLines += "}" |
| 294 | writeLinesIfNeeded(file, testOutLines) |
| 295 | testOutLines.clear() |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 296 | } |
| 297 | |
| 298 | private fun MarkdownTextReader.readUntil(marker: String): List<String> = |
| 299 | arrayListOf<String>().also { readUntilTo(marker, it) } |
| 300 | |
| 301 | private fun MarkdownTextReader.readUntilTo(marker: String, list: MutableList<String>) { |
| 302 | while (true) { |
| 303 | val line = readLine() ?: break |
| 304 | if (line.startsWith(marker)) break |
| 305 | list += line |
| 306 | } |
| 307 | } |
| 308 | |
Roman Elizarov | a5e653f | 2017-02-13 13:49:55 +0300 | [diff] [blame] | 309 | private inline fun <T> buildList(block: ArrayList<T>.() -> Unit): List<T> { |
| 310 | val result = arrayListOf<T>() |
| 311 | result.block() |
| 312 | return result |
| 313 | } |
| 314 | |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 315 | private fun requireSingleLine(directive: Directive) { |
| 316 | require(directive.singleLine) { "${directive.name} directive must end on the same line with '$DIRECTIVE_END'" } |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 317 | } |
| 318 | |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 319 | fun makeSectionRef(name: String): String = name.replace(' ', '-').replace(".", "").toLowerCase() |
| 320 | |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 321 | class Include(val regex: Regex, val lines: MutableList<String> = arrayListOf()) |
| 322 | |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 323 | class Directive( |
| 324 | val name: String, |
| 325 | val param: String, |
| 326 | val singleLine: Boolean |
| 327 | ) |
| 328 | |
| 329 | fun directive(line: String): Directive? { |
| 330 | if (!line.startsWith(DIRECTIVE_START)) return null |
| 331 | var s = line.substring(DIRECTIVE_START.length).trim() |
| 332 | val singleLine = s.endsWith(DIRECTIVE_END) |
| 333 | if (singleLine) s = s.substring(0, s.length - DIRECTIVE_END.length) |
| 334 | val i = s.indexOf(' ') |
| 335 | val name = if (i < 0) s else s.substring(0, i) |
| 336 | val param = if (i < 0) "" else s.substring(i).trim() |
| 337 | return Directive(name, param, singleLine) |
| 338 | } |
| 339 | |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 340 | class ApiRef(val line: Int, val name: String) |
| 341 | |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 342 | enum class MarkdownPart { PRE_TOC, TOC, POST_TOC } |
| 343 | |
| 344 | class MarkdownTextReader(r: Reader) : LineNumberReader(r) { |
Roman Elizarov | a5e653f | 2017-02-13 13:49:55 +0300 | [diff] [blame] | 345 | val inText = arrayListOf<String>() |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 346 | val preTocText = arrayListOf<String>() |
| 347 | val postTocText = arrayListOf<String>() |
| 348 | var markdownPart: MarkdownPart = MarkdownPart.PRE_TOC |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 349 | var skip = false |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 350 | |
Roman Elizarov | a5e653f | 2017-02-13 13:49:55 +0300 | [diff] [blame] | 351 | val outText: MutableList<String> get() = when (markdownPart) { |
| 352 | MarkdownPart.PRE_TOC -> preTocText |
| 353 | MarkdownPart.POST_TOC -> postTocText |
| 354 | else -> throw IllegalStateException("Wrong state: $markdownPart") |
| 355 | } |
| 356 | |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 357 | override fun readLine(): String? { |
| 358 | val line = super.readLine() ?: return null |
Roman Elizarov | a5e653f | 2017-02-13 13:49:55 +0300 | [diff] [blame] | 359 | inText += line |
| 360 | if (!skip && markdownPart != MarkdownPart.TOC) |
| 361 | outText += line |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 362 | return line |
| 363 | } |
| 364 | } |
| 365 | |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 366 | fun <T : LineNumberReader> File.withLineNumberReader(factory: (Reader) -> T, block: T.() -> Unit): T? { |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 367 | val reader = factory(reader()) |
| 368 | reader.use { |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 369 | try { |
| 370 | it.block() |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 371 | } catch (e: Exception) { |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 372 | println("ERROR: ${this@withLineNumberReader}: ${it.lineNumber}: ${e.message}") |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 373 | return null |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 374 | } |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 375 | } |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 376 | return reader |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 377 | } |
| 378 | |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 379 | fun File.withMarkdownTextReader(block: MarkdownTextReader.() -> Unit): MarkdownTextReader? = |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 380 | withLineNumberReader<MarkdownTextReader>(::MarkdownTextReader, block) |
| 381 | |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 382 | fun writeLinesIfNeeded(file: File, outLines: List<String>) { |
| 383 | val oldLines = try { |
| 384 | file.readLines() |
| 385 | } catch (e: IOException) { |
| 386 | emptyList<String>() |
| 387 | } |
| 388 | if (outLines != oldLines) writeLines(file, outLines) |
| 389 | } |
| 390 | |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 391 | fun writeLines(file: File, lines: List<String>) { |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 392 | println(" Writing $file ...") |
| 393 | file.parentFile?.mkdirs() |
| 394 | file.printWriter().use { out -> |
| 395 | lines.forEach { out.println(it) } |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 396 | } |
| 397 | } |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 398 | |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 399 | fun findModuleRootDir(name: String): String = |
| 400 | moduleRoots |
Vsevolod Tolstopyatov | 9c69279 | 2018-04-11 18:22:35 +0300 | [diff] [blame] | 401 | .map { "$it/$name" } |
| 402 | .firstOrNull { File("$it/$moduleMarker").exists() } |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 403 | ?: throw IllegalArgumentException("Module $name is not found in any of the module root dirs") |
| 404 | |
Roman Elizarov | d4dcbe2 | 2017-02-22 09:57:46 +0300 | [diff] [blame] | 405 | data class ApiIndexKey( |
| 406 | val docsRoot: String, |
| 407 | val pkg: String |
| 408 | ) |
| 409 | |
Vsevolod Tolstopyatov | 9c69279 | 2018-04-11 18:22:35 +0300 | [diff] [blame] | 410 | val apiIndexCache: MutableMap<ApiIndexKey, Map<String, List<String>>> = HashMap() |
Roman Elizarov | d4dcbe2 | 2017-02-22 09:57:46 +0300 | [diff] [blame] | 411 | |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 412 | val REF_LINE_REGEX = Regex("<a href=\"([a-z/.\\-]+)\">([a-zA-z.]+)</a>") |
| 413 | val INDEX_HTML = "/index.html" |
| 414 | val INDEX_MD = "/index.md" |
Roman Elizarov | 8839673 | 2017-09-27 21:30:47 +0300 | [diff] [blame] | 415 | val FUNCTIONS_SECTION_HEADER = "### Functions" |
| 416 | |
Vsevolod Tolstopyatov | 9c69279 | 2018-04-11 18:22:35 +0300 | [diff] [blame] | 417 | fun HashMap<String, MutableList<String>>.putUnambiguous(key: String, value: String) { |
Roman Elizarov | 8839673 | 2017-09-27 21:30:47 +0300 | [diff] [blame] | 418 | val oldValue = this[key] |
Vsevolod Tolstopyatov | 9c69279 | 2018-04-11 18:22:35 +0300 | [diff] [blame] | 419 | if (oldValue != null) { |
| 420 | oldValue.add(value) |
| 421 | put(key, oldValue) |
| 422 | } else { |
| 423 | put(key, mutableListOf(value)) |
| 424 | } |
Roman Elizarov | 8839673 | 2017-09-27 21:30:47 +0300 | [diff] [blame] | 425 | } |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 426 | |
Roman Elizarov | d4dcbe2 | 2017-02-22 09:57:46 +0300 | [diff] [blame] | 427 | fun loadApiIndex( |
| 428 | docsRoot: String, |
| 429 | path: String, |
| 430 | pkg: String, |
| 431 | namePrefix: String = "" |
Vsevolod Tolstopyatov | 9c69279 | 2018-04-11 18:22:35 +0300 | [diff] [blame] | 432 | ): Map<String, MutableList<String>>? { |
Roman Elizarov | e0c817d | 2017-02-10 10:22:01 +0300 | [diff] [blame] | 433 | val fileName = docsRoot + "/" + path + INDEX_MD |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 434 | val visited = mutableSetOf<String>() |
Vsevolod Tolstopyatov | 9c69279 | 2018-04-11 18:22:35 +0300 | [diff] [blame] | 435 | val map = HashMap<String, MutableList<String>>() |
Roman Elizarov | 8839673 | 2017-09-27 21:30:47 +0300 | [diff] [blame] | 436 | var inFunctionsSection = false |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 437 | File(fileName).withLineNumberReader<LineNumberReader>(::LineNumberReader) { |
| 438 | while (true) { |
| 439 | val line = readLine() ?: break |
Roman Elizarov | 8839673 | 2017-09-27 21:30:47 +0300 | [diff] [blame] | 440 | if (line == FUNCTIONS_SECTION_HEADER) inFunctionsSection = true |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 441 | val result = REF_LINE_REGEX.matchEntire(line) ?: continue |
Roman Elizarov | 8839673 | 2017-09-27 21:30:47 +0300 | [diff] [blame] | 442 | val link = result.groups[1]!!.value |
| 443 | if (link.startsWith("..")) continue // ignore cross-references |
| 444 | val absLink = path + "/" + link |
| 445 | var name = result.groups[2]!!.value |
| 446 | // a special disambiguation fix for pseudo-constructor functions |
| 447 | if (inFunctionsSection && name[0] in 'A'..'Z') name += "()" |
| 448 | val refName = namePrefix + name |
| 449 | val fqName = pkg + "." + refName |
| 450 | // Put short names for extensions on 3rd party classes (prefix is FQname of those classes) |
| 451 | if (namePrefix != "" && namePrefix[0] in 'a'..'z') map.putUnambiguous(name, absLink) |
| 452 | // Always put fully qualified names |
| 453 | map.putUnambiguous(refName, absLink) |
| 454 | map.putUnambiguous(fqName, absLink) |
| 455 | if (link.endsWith(INDEX_HTML)) { |
| 456 | if (visited.add(link)) { |
| 457 | val path2 = path + "/" + link.substring(0, link.length - INDEX_HTML.length) |
Roman Elizarov | d4dcbe2 | 2017-02-22 09:57:46 +0300 | [diff] [blame] | 458 | map += loadApiIndex(docsRoot, path2, pkg, refName + ".") |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 459 | ?: throw IllegalArgumentException("Failed to parse ${docsRoot + "/" + path2}") |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 460 | } |
| 461 | } |
| 462 | } |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 463 | } ?: return null // return null on failure |
Roman Elizarov | d4dcbe2 | 2017-02-22 09:57:46 +0300 | [diff] [blame] | 464 | return map |
| 465 | } |
| 466 | |
| 467 | fun processApiIndex( |
| 468 | siteRoot: String, |
| 469 | docsRoot: String, |
| 470 | pkg: String, |
| 471 | remainingApiRefNames: MutableSet<String> |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 472 | ): List<String>? { |
Roman Elizarov | d4dcbe2 | 2017-02-22 09:57:46 +0300 | [diff] [blame] | 473 | val key = ApiIndexKey(docsRoot, pkg) |
Roman Elizarov | 23f864e | 2017-03-03 19:57:47 +0300 | [diff] [blame] | 474 | val map = apiIndexCache.getOrPut(key, { |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 475 | print("Parsing API docs at $docsRoot/$pkg: ") |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 476 | val result = loadApiIndex(docsRoot, pkg, pkg) ?: return null // null on failure |
Roman Elizarov | 23f864e | 2017-03-03 19:57:47 +0300 | [diff] [blame] | 477 | println("${result.size} definitions") |
| 478 | result |
| 479 | }) |
Roman Elizarov | d4dcbe2 | 2017-02-22 09:57:46 +0300 | [diff] [blame] | 480 | val indexList = arrayListOf<String>() |
| 481 | val it = remainingApiRefNames.iterator() |
| 482 | while (it.hasNext()) { |
| 483 | val refName = it.next() |
| 484 | val refLink = map[refName] ?: continue |
Vsevolod Tolstopyatov | 9c69279 | 2018-04-11 18:22:35 +0300 | [diff] [blame] | 485 | if (refLink.size > 1) { |
| 486 | println("INFO: Ambiguous reference to [$refName]: $refLink, taking the shortest one") |
Roman Elizarov | 8839673 | 2017-09-27 21:30:47 +0300 | [diff] [blame] | 487 | } |
Vsevolod Tolstopyatov | 9c69279 | 2018-04-11 18:22:35 +0300 | [diff] [blame] | 488 | |
| 489 | val link = refLink.minBy { it.length } |
| 490 | indexList += "[$refName]: $siteRoot/$link" |
Roman Elizarov | d4dcbe2 | 2017-02-22 09:57:46 +0300 | [diff] [blame] | 491 | it.remove() |
| 492 | } |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 493 | return indexList |
Roman Elizarov | d4dcbe2 | 2017-02-22 09:57:46 +0300 | [diff] [blame] | 494 | } |