Roman Elizarov | f16fd27 | 2017-02-07 11:26:00 +0300 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2016-2017 JetBrains s.r.o. |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 17 | import java.io.File |
| 18 | import java.io.IOException |
| 19 | import java.io.LineNumberReader |
| 20 | import java.io.Reader |
| 21 | import java.util.* |
| 22 | import kotlin.properties.Delegates |
| 23 | |
| 24 | // --- props in knit.properties |
| 25 | |
| 26 | val knitProperties = ClassLoader.getSystemClassLoader() |
| 27 | .getResource("knit.properties").openStream().use { Properties().apply { load(it) } } |
| 28 | |
| 29 | val siteRoot = knitProperties.getProperty("site.root")!! |
| 30 | val moduleRoots = knitProperties.getProperty("module.roots").split(" ") |
| 31 | val moduleMarker = knitProperties.getProperty("module.marker")!! |
| 32 | val moduleDocs = knitProperties.getProperty("module.docs")!! |
| 33 | |
| 34 | // --- markdown syntax |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 35 | |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 36 | const val DIRECTIVE_START = "<!--- " |
| 37 | const val DIRECTIVE_END = "-->" |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 38 | |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 39 | const val TOC_DIRECTIVE = "TOC" |
| 40 | const val KNIT_DIRECTIVE = "KNIT" |
| 41 | const val INCLUDE_DIRECTIVE = "INCLUDE" |
| 42 | const val CLEAR_DIRECTIVE = "CLEAR" |
| 43 | const val TEST_DIRECTIVE = "TEST" |
Roman Elizarov | e0c817d | 2017-02-10 10:22:01 +0300 | [diff] [blame] | 44 | |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 45 | const val TEST_OUT_DIRECTIVE = "TEST_OUT" |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 46 | |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 47 | const val MODULE_DIRECTIVE = "MODULE" |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 48 | const val INDEX_DIRECTIVE = "INDEX" |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 49 | |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 50 | const val CODE_START = "```kotlin" |
| 51 | const val CODE_END = "```" |
| 52 | |
| 53 | const val TEST_START = "```text" |
| 54 | const val TEST_END = "```" |
| 55 | |
| 56 | const val SECTION_START = "##" |
| 57 | |
| 58 | const val PACKAGE_PREFIX = "package " |
| 59 | const val STARTS_WITH_PREDICATE = "STARTS_WITH" |
Roman Elizarov | 1e45960 | 2017-02-27 11:05:17 +0300 | [diff] [blame] | 60 | const val ARBITRARY_TIME_PREDICATE = "ARBITRARY_TIME" |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 61 | const val FLEXIBLE_TIME_PREDICATE = "FLEXIBLE_TIME" |
| 62 | const val FLEXIBLE_THREAD_PREDICATE = "FLEXIBLE_THREAD" |
| 63 | const val LINES_START_UNORDERED_PREDICATE = "LINES_START_UNORDERED" |
| 64 | const val LINES_START_PREDICATE = "LINES_START" |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 65 | |
Roman Elizarov | 8839673 | 2017-09-27 21:30:47 +0300 | [diff] [blame] | 66 | val API_REF_REGEX = Regex("(^|[ \\]])\\[([A-Za-z0-9_().]+)\\]($|[^\\[\\(])") |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 67 | |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 68 | fun main(args: Array<String>) { |
Roman Elizarov | a5e653f | 2017-02-13 13:49:55 +0300 | [diff] [blame] | 69 | if (args.isEmpty()) { |
| 70 | println("Usage: Knit <markdown-files>") |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 71 | return |
| 72 | } |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 73 | args.forEach { |
| 74 | if (!knit(it)) System.exit(1) // abort on first error with error exit code |
| 75 | } |
Roman Elizarov | a5e653f | 2017-02-13 13:49:55 +0300 | [diff] [blame] | 76 | } |
| 77 | |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 78 | fun knit(markdownFileName: String): Boolean { |
Roman Elizarov | a5e653f | 2017-02-13 13:49:55 +0300 | [diff] [blame] | 79 | println("*** Reading $markdownFileName") |
| 80 | val markdownFile = File(markdownFileName) |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 81 | val tocLines = arrayListOf<String>() |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 82 | var knitRegex: Regex? = null |
| 83 | val includes = arrayListOf<Include>() |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 84 | val codeLines = arrayListOf<String>() |
| 85 | val testLines = arrayListOf<String>() |
| 86 | var testOut: String? = null |
| 87 | val testOutLines = arrayListOf<String>() |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 88 | var lastPgk: String? = null |
Roman Elizarov | 23f864e | 2017-03-03 19:57:47 +0300 | [diff] [blame] | 89 | val files = mutableSetOf<File>() |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 90 | val allApiRefs = arrayListOf<ApiRef>() |
| 91 | val remainingApiRefNames = mutableSetOf<String>() |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 92 | var moduleName: String by Delegates.notNull() |
| 93 | var docsRoot: String by Delegates.notNull() |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 94 | // read markdown file |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 95 | var putBackLine: String? = null |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 96 | val markdown = markdownFile.withMarkdownTextReader { |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 97 | mainLoop@ while (true) { |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 98 | val inLine = putBackLine ?: readLine() ?: break |
| 99 | putBackLine = null |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 100 | val directive = directive(inLine) |
| 101 | if (directive != null && markdownPart == MarkdownPart.TOC) { |
| 102 | markdownPart = MarkdownPart.POST_TOC |
| 103 | postTocText += inLine |
| 104 | } |
| 105 | when (directive?.name) { |
| 106 | TOC_DIRECTIVE -> { |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 107 | requireSingleLine(directive) |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 108 | require(directive.param.isEmpty()) { "$TOC_DIRECTIVE directive must not have parameters" } |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 109 | require(markdownPart == MarkdownPart.PRE_TOC) { "Only one TOC directive is supported" } |
| 110 | markdownPart = MarkdownPart.TOC |
| 111 | } |
| 112 | KNIT_DIRECTIVE -> { |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 113 | requireSingleLine(directive) |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 114 | require(!directive.param.isEmpty()) { "$KNIT_DIRECTIVE directive must include regex parameter" } |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 115 | require(knitRegex == null) { "Only one KNIT directive is supported"} |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 116 | knitRegex = Regex("\\((" + directive.param + ")\\)") |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 117 | continue@mainLoop |
| 118 | } |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 119 | INCLUDE_DIRECTIVE -> { |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 120 | if (directive.param.isEmpty()) { |
| 121 | require(!directive.singleLine) { "$INCLUDE_DIRECTIVE directive without parameters must not be single line" } |
| 122 | readUntilTo(DIRECTIVE_END, codeLines) |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 123 | } else { |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 124 | val include = Include(Regex(directive.param)) |
| 125 | if (directive.singleLine) { |
| 126 | include.lines += codeLines |
| 127 | codeLines.clear() |
| 128 | } else { |
| 129 | readUntilTo(DIRECTIVE_END, include.lines) |
| 130 | } |
| 131 | includes += include |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 132 | } |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 133 | continue@mainLoop |
| 134 | } |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 135 | CLEAR_DIRECTIVE -> { |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 136 | requireSingleLine(directive) |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 137 | require(directive.param.isEmpty()) { "$CLEAR_DIRECTIVE directive must not have parameters" } |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 138 | codeLines.clear() |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 139 | continue@mainLoop |
| 140 | } |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 141 | TEST_OUT_DIRECTIVE -> { |
| 142 | require(!directive.param.isEmpty()) { "$TEST_OUT_DIRECTIVE directive must include file name parameter" } |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 143 | flushTestOut(markdownFile.parentFile, testOut, testOutLines) |
| 144 | testOut = directive.param |
| 145 | readUntil(DIRECTIVE_END).forEach { testOutLines += it } |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 146 | } |
| 147 | TEST_DIRECTIVE -> { |
| 148 | require(lastPgk != null) { "'$PACKAGE_PREFIX' prefix was not found in emitted code"} |
| 149 | require(testOut != null) { "$TEST_OUT_DIRECTIVE directive was not specified" } |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 150 | val predicate = directive.param |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 151 | if (testLines.isEmpty()) { |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 152 | if (directive.singleLine) { |
| 153 | require(!predicate.isEmpty()) { "$TEST_OUT_DIRECTIVE must be preceded by $TEST_START block or contain test predicate"} |
| 154 | } else |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 155 | testLines += readUntil(DIRECTIVE_END) |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 156 | } else { |
| 157 | requireSingleLine(directive) |
| 158 | } |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 159 | makeTest(testOutLines, lastPgk!!, testLines, predicate) |
| 160 | testLines.clear() |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 161 | } |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 162 | MODULE_DIRECTIVE -> { |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 163 | requireSingleLine(directive) |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 164 | moduleName = directive.param |
| 165 | docsRoot = findModuleRootDir(moduleName) + "/" + moduleDocs + "/" + moduleName |
Roman Elizarov | e0c817d | 2017-02-10 10:22:01 +0300 | [diff] [blame] | 166 | } |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 167 | INDEX_DIRECTIVE -> { |
| 168 | requireSingleLine(directive) |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 169 | val indexLines = processApiIndex(siteRoot + "/" + moduleName, docsRoot, directive.param, remainingApiRefNames) |
| 170 | ?: throw IllegalArgumentException("Failed to load index for ${directive.param}") |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 171 | skip = true |
| 172 | while (true) { |
| 173 | val skipLine = readLine() ?: break@mainLoop |
| 174 | if (directive(skipLine) != null) { |
| 175 | putBackLine = skipLine |
| 176 | break |
| 177 | } |
| 178 | } |
| 179 | skip = false |
Roman Elizarov | a5e653f | 2017-02-13 13:49:55 +0300 | [diff] [blame] | 180 | outText += indexLines |
| 181 | outText += putBackLine!! |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 182 | } |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 183 | } |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 184 | if (inLine.startsWith(CODE_START)) { |
Roman Elizarov | f724f6e | 2017-04-07 18:06:22 +0300 | [diff] [blame] | 185 | require(testOut == null || testLines.isEmpty()) { "Previous test was not emitted with $TEST_DIRECTIVE" } |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 186 | codeLines += "" |
| 187 | readUntilTo(CODE_END, codeLines) |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 188 | continue@mainLoop |
| 189 | } |
| 190 | if (inLine.startsWith(TEST_START)) { |
Roman Elizarov | f724f6e | 2017-04-07 18:06:22 +0300 | [diff] [blame] | 191 | require(testOut == null || testLines.isEmpty()) { "Previous test was not emitted with $TEST_DIRECTIVE" } |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 192 | readUntilTo(TEST_END, testLines) |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 193 | continue@mainLoop |
| 194 | } |
| 195 | if (inLine.startsWith(SECTION_START) && markdownPart == MarkdownPart.POST_TOC) { |
| 196 | val i = inLine.indexOf(' ') |
| 197 | require(i >= 2) { "Invalid section start" } |
| 198 | val name = inLine.substring(i + 1).trim() |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 199 | tocLines += " ".repeat(i - 2) + "* [$name](#${makeSectionRef(name)})" |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 200 | continue@mainLoop |
| 201 | } |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 202 | for (match in API_REF_REGEX.findAll(inLine)) { |
| 203 | val apiRef = ApiRef(lineNumber, match.groups[2]!!.value) |
| 204 | allApiRefs += apiRef |
| 205 | remainingApiRefNames += apiRef.name |
| 206 | } |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 207 | knitRegex?.find(inLine)?.let { knitMatch -> |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 208 | val fileName = knitMatch.groups[1]!!.value |
Roman Elizarov | 23f864e | 2017-03-03 19:57:47 +0300 | [diff] [blame] | 209 | val file = File(markdownFile.parentFile, fileName) |
| 210 | require(files.add(file)) { "Duplicate file: $file"} |
| 211 | println("Knitting $file ...") |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 212 | val outLines = arrayListOf<String>() |
| 213 | for (include in includes) { |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 214 | val includeMatch = include.regex.matchEntire(fileName) ?: continue |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 215 | include.lines.forEach { includeLine -> |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 216 | val line = makeReplacements(includeLine, includeMatch) |
| 217 | if (line.startsWith(PACKAGE_PREFIX)) |
| 218 | lastPgk = line.substring(PACKAGE_PREFIX.length).trim() |
| 219 | outLines += line |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 220 | } |
| 221 | } |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 222 | outLines += codeLines |
| 223 | codeLines.clear() |
| 224 | writeLinesIfNeeded(file, outLines) |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 225 | } |
| 226 | } |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 227 | } ?: return false // false when failed |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 228 | // update markdown file with toc |
Roman Elizarov | a5e653f | 2017-02-13 13:49:55 +0300 | [diff] [blame] | 229 | val newLines = buildList<String> { |
| 230 | addAll(markdown.preTocText) |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 231 | if (!tocLines.isEmpty()) { |
Roman Elizarov | a5e653f | 2017-02-13 13:49:55 +0300 | [diff] [blame] | 232 | add("") |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 233 | addAll(tocLines) |
Roman Elizarov | a5e653f | 2017-02-13 13:49:55 +0300 | [diff] [blame] | 234 | add("") |
| 235 | } |
| 236 | addAll(markdown.postTocText) |
| 237 | } |
| 238 | if (newLines != markdown.inText) writeLines(markdownFile, newLines) |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 239 | // check apiRefs |
| 240 | for (apiRef in allApiRefs) { |
| 241 | if (apiRef.name in remainingApiRefNames) { |
| 242 | println("WARNING: $markdownFile: ${apiRef.line}: Broken reference to [${apiRef.name}]") |
| 243 | } |
| 244 | } |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 245 | // write test output |
| 246 | flushTestOut(markdownFile.parentFile, testOut, testOutLines) |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 247 | return true |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 248 | } |
| 249 | |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 250 | fun makeTest(testOutLines: MutableList<String>, pgk: String, test: List<String>, predicate: String) { |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 251 | val funName = buildString { |
| 252 | var cap = true |
| 253 | for (c in pgk) { |
| 254 | if (c == '.') { |
| 255 | cap = true |
| 256 | } else { |
| 257 | append(if (cap) c.toUpperCase() else c) |
| 258 | cap = false |
| 259 | } |
| 260 | } |
| 261 | } |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 262 | testOutLines += "" |
| 263 | testOutLines += " @Test" |
| 264 | testOutLines += " fun test$funName() {" |
Roman Elizarov | ba0c004 | 2017-07-12 13:01:17 +0300 | [diff] [blame] | 265 | val prefix = " test(\"$funName\") { $pgk.main(emptyArray()) }" |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 266 | when (predicate) { |
| 267 | "" -> makeTestLines(testOutLines, prefix, "verifyLines", test) |
| 268 | STARTS_WITH_PREDICATE -> makeTestLines(testOutLines, prefix, "verifyLinesStartWith", test) |
| 269 | ARBITRARY_TIME_PREDICATE -> makeTestLines(testOutLines, prefix, "verifyLinesArbitraryTime", test) |
| 270 | FLEXIBLE_TIME_PREDICATE -> makeTestLines(testOutLines, prefix, "verifyLinesFlexibleTime", test) |
| 271 | FLEXIBLE_THREAD_PREDICATE -> makeTestLines(testOutLines, prefix, "verifyLinesFlexibleThread", test) |
| 272 | LINES_START_UNORDERED_PREDICATE -> makeTestLines(testOutLines, prefix, "verifyLinesStartUnordered", test) |
| 273 | LINES_START_PREDICATE -> makeTestLines(testOutLines, prefix, "verifyLinesStart", test) |
| 274 | else -> { |
| 275 | testOutLines += prefix + ".also { lines ->" |
| 276 | testOutLines += " check($predicate)" |
| 277 | testOutLines += " }" |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 278 | } |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 279 | } |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 280 | testOutLines += " }" |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 281 | } |
| 282 | |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 283 | private fun makeTestLines(testOutLines: MutableList<String>, prefix: String, method: String, test: List<String>) { |
| 284 | testOutLines += "$prefix.$method(" |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 285 | for ((index, testLine) in test.withIndex()) { |
| 286 | val commaOpt = if (index < test.size - 1) "," else "" |
| 287 | val escapedLine = testLine.replace("\"", "\\\"") |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 288 | testOutLines += " \"$escapedLine\"$commaOpt" |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 289 | } |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 290 | testOutLines += " )" |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 291 | } |
| 292 | |
| 293 | private fun makeReplacements(line: String, match: MatchResult): String { |
| 294 | var result = line |
| 295 | for ((id, group) in match.groups.withIndex()) { |
| 296 | if (group != null) |
| 297 | result = result.replace("\$\$$id", group.value) |
| 298 | } |
| 299 | return result |
| 300 | } |
| 301 | |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 302 | private fun flushTestOut(parentDir: File?, testOut: String?, testOutLines: MutableList<String>) { |
| 303 | if (testOut == null) return |
| 304 | val file = File(parentDir, testOut) |
| 305 | testOutLines += "}" |
| 306 | writeLinesIfNeeded(file, testOutLines) |
| 307 | testOutLines.clear() |
Roman Elizarov | 731f0ad | 2017-02-22 20:48:45 +0300 | [diff] [blame] | 308 | } |
| 309 | |
| 310 | private fun MarkdownTextReader.readUntil(marker: String): List<String> = |
| 311 | arrayListOf<String>().also { readUntilTo(marker, it) } |
| 312 | |
| 313 | private fun MarkdownTextReader.readUntilTo(marker: String, list: MutableList<String>) { |
| 314 | while (true) { |
| 315 | val line = readLine() ?: break |
| 316 | if (line.startsWith(marker)) break |
| 317 | list += line |
| 318 | } |
| 319 | } |
| 320 | |
Roman Elizarov | a5e653f | 2017-02-13 13:49:55 +0300 | [diff] [blame] | 321 | private inline fun <T> buildList(block: ArrayList<T>.() -> Unit): List<T> { |
| 322 | val result = arrayListOf<T>() |
| 323 | result.block() |
| 324 | return result |
| 325 | } |
| 326 | |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 327 | private fun requireSingleLine(directive: Directive) { |
| 328 | require(directive.singleLine) { "${directive.name} directive must end on the same line with '$DIRECTIVE_END'" } |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 329 | } |
| 330 | |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 331 | fun makeSectionRef(name: String): String = name.replace(' ', '-').replace(".", "").toLowerCase() |
| 332 | |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 333 | class Include(val regex: Regex, val lines: MutableList<String> = arrayListOf()) |
| 334 | |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 335 | class Directive( |
| 336 | val name: String, |
| 337 | val param: String, |
| 338 | val singleLine: Boolean |
| 339 | ) |
| 340 | |
| 341 | fun directive(line: String): Directive? { |
| 342 | if (!line.startsWith(DIRECTIVE_START)) return null |
| 343 | var s = line.substring(DIRECTIVE_START.length).trim() |
| 344 | val singleLine = s.endsWith(DIRECTIVE_END) |
| 345 | if (singleLine) s = s.substring(0, s.length - DIRECTIVE_END.length) |
| 346 | val i = s.indexOf(' ') |
| 347 | val name = if (i < 0) s else s.substring(0, i) |
| 348 | val param = if (i < 0) "" else s.substring(i).trim() |
| 349 | return Directive(name, param, singleLine) |
| 350 | } |
| 351 | |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 352 | class ApiRef(val line: Int, val name: String) |
| 353 | |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 354 | enum class MarkdownPart { PRE_TOC, TOC, POST_TOC } |
| 355 | |
| 356 | class MarkdownTextReader(r: Reader) : LineNumberReader(r) { |
Roman Elizarov | a5e653f | 2017-02-13 13:49:55 +0300 | [diff] [blame] | 357 | val inText = arrayListOf<String>() |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 358 | val preTocText = arrayListOf<String>() |
| 359 | val postTocText = arrayListOf<String>() |
| 360 | var markdownPart: MarkdownPart = MarkdownPart.PRE_TOC |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 361 | var skip = false |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 362 | |
Roman Elizarov | a5e653f | 2017-02-13 13:49:55 +0300 | [diff] [blame] | 363 | val outText: MutableList<String> get() = when (markdownPart) { |
| 364 | MarkdownPart.PRE_TOC -> preTocText |
| 365 | MarkdownPart.POST_TOC -> postTocText |
| 366 | else -> throw IllegalStateException("Wrong state: $markdownPart") |
| 367 | } |
| 368 | |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 369 | override fun readLine(): String? { |
| 370 | val line = super.readLine() ?: return null |
Roman Elizarov | a5e653f | 2017-02-13 13:49:55 +0300 | [diff] [blame] | 371 | inText += line |
| 372 | if (!skip && markdownPart != MarkdownPart.TOC) |
| 373 | outText += line |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 374 | return line |
| 375 | } |
| 376 | } |
| 377 | |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 378 | fun <T : LineNumberReader> File.withLineNumberReader(factory: (Reader) -> T, block: T.() -> Unit): T? { |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 379 | val reader = factory(reader()) |
| 380 | reader.use { |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 381 | try { |
| 382 | it.block() |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 383 | } catch (e: Exception) { |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 384 | println("ERROR: ${this@withLineNumberReader}: ${it.lineNumber}: ${e.message}") |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 385 | return null |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 386 | } |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 387 | } |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 388 | return reader |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 389 | } |
| 390 | |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 391 | fun File.withMarkdownTextReader(block: MarkdownTextReader.() -> Unit): MarkdownTextReader? = |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 392 | withLineNumberReader<MarkdownTextReader>(::MarkdownTextReader, block) |
| 393 | |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 394 | fun writeLinesIfNeeded(file: File, outLines: List<String>) { |
| 395 | val oldLines = try { |
| 396 | file.readLines() |
| 397 | } catch (e: IOException) { |
| 398 | emptyList<String>() |
| 399 | } |
| 400 | if (outLines != oldLines) writeLines(file, outLines) |
| 401 | } |
| 402 | |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 403 | fun writeLines(file: File, lines: List<String>) { |
Roman Elizarov | fa7723e | 2017-02-06 11:17:51 +0300 | [diff] [blame] | 404 | println(" Writing $file ...") |
| 405 | file.parentFile?.mkdirs() |
| 406 | file.printWriter().use { out -> |
| 407 | lines.forEach { out.println(it) } |
Roman Elizarov | b3d55a5 | 2017-02-03 12:47:21 +0300 | [diff] [blame] | 408 | } |
| 409 | } |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 410 | |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 411 | fun findModuleRootDir(name: String): String = |
| 412 | moduleRoots |
Vsevolod Tolstopyatov | 9c69279 | 2018-04-11 18:22:35 +0300 | [diff] [blame] | 413 | .map { "$it/$name" } |
| 414 | .firstOrNull { File("$it/$moduleMarker").exists() } |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 415 | ?: throw IllegalArgumentException("Module $name is not found in any of the module root dirs") |
| 416 | |
Roman Elizarov | d4dcbe2 | 2017-02-22 09:57:46 +0300 | [diff] [blame] | 417 | data class ApiIndexKey( |
| 418 | val docsRoot: String, |
| 419 | val pkg: String |
| 420 | ) |
| 421 | |
Vsevolod Tolstopyatov | 9c69279 | 2018-04-11 18:22:35 +0300 | [diff] [blame] | 422 | val apiIndexCache: MutableMap<ApiIndexKey, Map<String, List<String>>> = HashMap() |
Roman Elizarov | d4dcbe2 | 2017-02-22 09:57:46 +0300 | [diff] [blame] | 423 | |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 424 | val REF_LINE_REGEX = Regex("<a href=\"([a-z/.\\-]+)\">([a-zA-z.]+)</a>") |
| 425 | val INDEX_HTML = "/index.html" |
| 426 | val INDEX_MD = "/index.md" |
Roman Elizarov | 8839673 | 2017-09-27 21:30:47 +0300 | [diff] [blame] | 427 | val FUNCTIONS_SECTION_HEADER = "### Functions" |
| 428 | |
Vsevolod Tolstopyatov | 9c69279 | 2018-04-11 18:22:35 +0300 | [diff] [blame] | 429 | fun HashMap<String, MutableList<String>>.putUnambiguous(key: String, value: String) { |
Roman Elizarov | 8839673 | 2017-09-27 21:30:47 +0300 | [diff] [blame] | 430 | val oldValue = this[key] |
Vsevolod Tolstopyatov | 9c69279 | 2018-04-11 18:22:35 +0300 | [diff] [blame] | 431 | if (oldValue != null) { |
| 432 | oldValue.add(value) |
| 433 | put(key, oldValue) |
| 434 | } else { |
| 435 | put(key, mutableListOf(value)) |
| 436 | } |
Roman Elizarov | 8839673 | 2017-09-27 21:30:47 +0300 | [diff] [blame] | 437 | } |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 438 | |
Roman Elizarov | d4dcbe2 | 2017-02-22 09:57:46 +0300 | [diff] [blame] | 439 | fun loadApiIndex( |
| 440 | docsRoot: String, |
| 441 | path: String, |
| 442 | pkg: String, |
| 443 | namePrefix: String = "" |
Vsevolod Tolstopyatov | 9c69279 | 2018-04-11 18:22:35 +0300 | [diff] [blame] | 444 | ): Map<String, MutableList<String>>? { |
Roman Elizarov | e0c817d | 2017-02-10 10:22:01 +0300 | [diff] [blame] | 445 | val fileName = docsRoot + "/" + path + INDEX_MD |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 446 | val visited = mutableSetOf<String>() |
Vsevolod Tolstopyatov | 9c69279 | 2018-04-11 18:22:35 +0300 | [diff] [blame] | 447 | val map = HashMap<String, MutableList<String>>() |
Roman Elizarov | 8839673 | 2017-09-27 21:30:47 +0300 | [diff] [blame] | 448 | var inFunctionsSection = false |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 449 | File(fileName).withLineNumberReader<LineNumberReader>(::LineNumberReader) { |
| 450 | while (true) { |
| 451 | val line = readLine() ?: break |
Roman Elizarov | 8839673 | 2017-09-27 21:30:47 +0300 | [diff] [blame] | 452 | if (line == FUNCTIONS_SECTION_HEADER) inFunctionsSection = true |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 453 | val result = REF_LINE_REGEX.matchEntire(line) ?: continue |
Roman Elizarov | 8839673 | 2017-09-27 21:30:47 +0300 | [diff] [blame] | 454 | val link = result.groups[1]!!.value |
| 455 | if (link.startsWith("..")) continue // ignore cross-references |
| 456 | val absLink = path + "/" + link |
| 457 | var name = result.groups[2]!!.value |
| 458 | // a special disambiguation fix for pseudo-constructor functions |
| 459 | if (inFunctionsSection && name[0] in 'A'..'Z') name += "()" |
| 460 | val refName = namePrefix + name |
| 461 | val fqName = pkg + "." + refName |
| 462 | // Put short names for extensions on 3rd party classes (prefix is FQname of those classes) |
| 463 | if (namePrefix != "" && namePrefix[0] in 'a'..'z') map.putUnambiguous(name, absLink) |
| 464 | // Always put fully qualified names |
| 465 | map.putUnambiguous(refName, absLink) |
| 466 | map.putUnambiguous(fqName, absLink) |
| 467 | if (link.endsWith(INDEX_HTML)) { |
| 468 | if (visited.add(link)) { |
| 469 | val path2 = path + "/" + link.substring(0, link.length - INDEX_HTML.length) |
Roman Elizarov | d4dcbe2 | 2017-02-22 09:57:46 +0300 | [diff] [blame] | 470 | map += loadApiIndex(docsRoot, path2, pkg, refName + ".") |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 471 | ?: throw IllegalArgumentException("Failed to parse ${docsRoot + "/" + path2}") |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 472 | } |
| 473 | } |
| 474 | } |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 475 | } ?: return null // return null on failure |
Roman Elizarov | d4dcbe2 | 2017-02-22 09:57:46 +0300 | [diff] [blame] | 476 | return map |
| 477 | } |
| 478 | |
| 479 | fun processApiIndex( |
| 480 | siteRoot: String, |
| 481 | docsRoot: String, |
| 482 | pkg: String, |
| 483 | remainingApiRefNames: MutableSet<String> |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 484 | ): List<String>? { |
Roman Elizarov | d4dcbe2 | 2017-02-22 09:57:46 +0300 | [diff] [blame] | 485 | val key = ApiIndexKey(docsRoot, pkg) |
Roman Elizarov | 23f864e | 2017-03-03 19:57:47 +0300 | [diff] [blame] | 486 | val map = apiIndexCache.getOrPut(key, { |
Roman Elizarov | 8a4a8e1 | 2017-03-09 19:52:58 +0300 | [diff] [blame] | 487 | print("Parsing API docs at $docsRoot/$pkg: ") |
Roman Elizarov | e7e2ad1 | 2017-05-17 14:47:31 +0300 | [diff] [blame] | 488 | val result = loadApiIndex(docsRoot, pkg, pkg) ?: return null // null on failure |
Roman Elizarov | 23f864e | 2017-03-03 19:57:47 +0300 | [diff] [blame] | 489 | println("${result.size} definitions") |
| 490 | result |
| 491 | }) |
Roman Elizarov | d4dcbe2 | 2017-02-22 09:57:46 +0300 | [diff] [blame] | 492 | val indexList = arrayListOf<String>() |
| 493 | val it = remainingApiRefNames.iterator() |
| 494 | while (it.hasNext()) { |
| 495 | val refName = it.next() |
| 496 | val refLink = map[refName] ?: continue |
Vsevolod Tolstopyatov | 9c69279 | 2018-04-11 18:22:35 +0300 | [diff] [blame] | 497 | if (refLink.size > 1) { |
| 498 | println("INFO: Ambiguous reference to [$refName]: $refLink, taking the shortest one") |
Roman Elizarov | 8839673 | 2017-09-27 21:30:47 +0300 | [diff] [blame] | 499 | } |
Vsevolod Tolstopyatov | 9c69279 | 2018-04-11 18:22:35 +0300 | [diff] [blame] | 500 | |
| 501 | val link = refLink.minBy { it.length } |
| 502 | indexList += "[$refName]: $siteRoot/$link" |
Roman Elizarov | d4dcbe2 | 2017-02-22 09:57:46 +0300 | [diff] [blame] | 503 | it.remove() |
| 504 | } |
Roman Elizarov | 419a6c8 | 2017-02-09 18:36:22 +0300 | [diff] [blame] | 505 | return indexList |
Roman Elizarov | d4dcbe2 | 2017-02-22 09:57:46 +0300 | [diff] [blame] | 506 | } |