diff options
author | Cary Clark <caryclark@skia.org> | 2017-07-28 11:04:54 -0400 |
---|---|---|
committer | Skia Commit-Bot <skia-commit-bot@chromium.org> | 2017-07-28 15:30:38 +0000 |
commit | 8032b983faaa8c76f81bf3cf028e9c64f4635478 (patch) | |
tree | ed3be061ff02a99dab1b3e443d48b7f5c906417e /tools | |
parent | acaa607328fb0dfac0894d4a2fcdead520e696b3 (diff) |
bookmaker initial checkin
bookmaker is a tool that generates documentation
backends from a canonical markup. Documentation for
bookmaker itself is evolving at docs/usingBookmaker.bmh,
which is visible online at skia.org/user/api/bmh_usingBookmaker
Change-Id: Ic76ddf29134895b5c2ebfbc84603e40ff08caf09
Reviewed-on: https://skia-review.googlesource.com/28000
Commit-Queue: Cary Clark <caryclark@google.com>
Reviewed-by: Cary Clark <caryclark@google.com>
Diffstat (limited to 'tools')
-rw-r--r-- | tools/bookmaker/bookmaker.cpp | 2198 | ||||
-rw-r--r-- | tools/bookmaker/bookmaker.h | 1844 | ||||
-rw-r--r-- | tools/bookmaker/fiddleParser.cpp | 231 | ||||
-rw-r--r-- | tools/bookmaker/includeParser.cpp | 1733 | ||||
-rw-r--r-- | tools/bookmaker/includeWriter.cpp | 1272 | ||||
-rw-r--r-- | tools/bookmaker/mdOut.cpp | 929 | ||||
-rw-r--r-- | tools/bookmaker/parserCommon.cpp | 51 | ||||
-rw-r--r-- | tools/bookmaker/spellCheck.cpp | 455 |
8 files changed, 8713 insertions, 0 deletions
diff --git a/tools/bookmaker/bookmaker.cpp b/tools/bookmaker/bookmaker.cpp new file mode 100644 index 0000000000..3b67663000 --- /dev/null +++ b/tools/bookmaker/bookmaker.cpp @@ -0,0 +1,2198 @@ +/* + * Copyright 2017 Google Inc. + * + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include "bookmaker.h" + +#include "SkCommandLineFlags.h" +#include "SkOSFile.h" +#include "SkOSPath.h" + + +/* recipe for generating timestamps for existing doxygen comments +find include/core -type f -name '*.h' -print -exec git blame {} \; > ~/all.blame.txt + +space table better for Constants +should Return be on same line as 'Return Value'? +remove anonymous header, e.g. Enum SkPaint::::anonymous_2 +Text Encoding anchors in paragraph are echoed instead of being linked to anchor names + also should not point to 'undocumented' since they are resolvable links +#Member lost all formatting +inconsistent use of capitalization in #Param +#List needs '# content ##', formatting +consts like enum members need fully qualfied refs to make a valid link +enum comments should be disallowed unless after #Enum and before first #Const + ... or, should look for enum comments in other places + +// in includeWriter.cpp +lf preceding #A is ignored + +Text_Size should become SkPaint's text size if root is not Paint? +100 column limit done manually -- either error or rewrap + +SkPaint.bmh line 22: +Insert 'the' after 'regardless of' ? +somewhat intentional. Imagine SkPaint::kXXX is 'Joe'. Then it shouldn't read 'regardless +of the Joe setting.' To make that work as a proper pronoun, maybe it should read: +'regardless of SkPaint's kAntiAlias_Flag setting or 'regardless of SkPaint's anti-alias setting'. +It's the way it is so that SkPaint::kAntiAlias_Flag can be a link to the definition. +Its awkwardness is compounded because this description is technically outside of 'class SkPaint' +so a reference to kAntiAlias_Flag by itself doesn't know that it is defined inside SkPaint, +but that's a detail I could work around. + +SkPaint.bmh line 319, 400, 444 +more complications I haven't figured out. I don't know when or how to pluralize +references. This should be "objects' reference counts" probably, but then +I lose the link to SkRefCnt. + +SkPaint.bmh line 2074 +arcs at front of sentence not capitalized + +SkPaint.bmh line 2639 +I'd argue that 'fill path' is OK, in that is it the path that will fill, not the path +that has already been filled. I see the awkwardness though, and will add it to my bug list. + +check for function name in its own description + +multiple line #Param / #Return only copies first line? + +rework underlinethickness / strikeout thickness + +getTextIntercepts lost underline comment + */ + +static string normalized_name(string name) { + string normalizedName = name; + std::replace(normalizedName.begin(), normalizedName.end(), '-', '_'); + do { + size_t doubleColon = normalizedName.find("::", 0); + if (string::npos == doubleColon) { + break; + } + normalizedName = normalizedName.substr(0, doubleColon) + + '_' + normalizedName.substr(doubleColon + 2); + } while (true); + return normalizedName; +} + +static size_t count_indent(const string& text, size_t test, size_t end) { + size_t result = test; + while (test < end) { + if (' ' != text[test]) { + break; + } + ++test; + } + return test - result; +} + +static void add_code(const string& text, int pos, int end, + size_t outIndent, size_t textIndent, string& example) { + do { + // fix this to move whole paragraph in, out, but preserve doc indent + int nextIndent = count_indent(text, pos, end); + size_t len = text.find('\n', pos); + if (string::npos == len) { + len = end; + } + if ((size_t) (pos + nextIndent) < len) { + size_t indent = outIndent + nextIndent; + SkASSERT(indent >= textIndent); + indent -= textIndent; + for (size_t index = 0; index < indent; ++index) { + example += ' '; + } + pos += nextIndent; + while ((size_t) pos < len) { + example += '"' == text[pos] ? "\\\"" : + '\\' == text[pos] ? "\\\\" : + text.substr(pos, 1); + ++pos; + } + example += "\\n"; + } else { + pos += nextIndent; + } + if ('\n' == text[pos]) { + ++pos; + } + } while (pos < end); +} + +// fixme: this will need to be more complicated to handle all of Skia +// for now, just handle paint -- maybe fiddle will loosen naming restrictions +void Definition::setCanonicalFiddle() { + fMethodType = Definition::MethodType::kNone; + size_t doubleColons = fName.find("::", 0); + SkASSERT(string::npos != doubleColons); + string result = fName.substr(0, doubleColons) + "_"; + doubleColons += 2; + if (string::npos != fName.find('~', doubleColons)) { + fMethodType = Definition::MethodType::kDestructor; + result += "destructor"; + } else { + bool isMove = string::npos != fName.find("&&", doubleColons); + const char operatorStr[] = "operator"; + size_t opPos = fName.find(operatorStr, doubleColons); + if (string::npos != opPos) { + fMethodType = Definition::MethodType::kOperator; + opPos += sizeof(operatorStr) - 1; + if ('!' == fName[opPos]) { + SkASSERT('=' == fName[opPos + 1]); + result += "not_equal_operator"; + } else if ('=' == fName[opPos]) { + if ('(' == fName[opPos + 1]) { + result += isMove ? "move_" : "copy_"; + result += "assignment_operator"; + } else { + SkASSERT('=' == fName[opPos + 1]); + result += "equal_operator"; + } + } else { + SkASSERT(0); // todo: incomplete + } + } else if (string::npos != fName.find("()", doubleColons)) { + if (isupper(fName[doubleColons])) { + fMethodType = Definition::MethodType::kConstructor; + result += "empty_constructor"; + } else { + result += fName.substr(doubleColons, fName.length() - doubleColons - 2); + } + } else { + size_t comma = fName.find(',', doubleColons); + size_t openParen = fName.find('(', doubleColons); + if (string::npos == comma && string::npos != openParen) { + fMethodType = Definition::MethodType::kConstructor; + result += isMove ? "move_" : "copy_"; + result += "constructor"; + } else if (string::npos == openParen) { + result += fName.substr(doubleColons); + } else { + fMethodType = Definition::MethodType::kConstructor; + // name them by their param types, e.g. SkCanvas__int_int_const_SkSurfaceProps_star + SkASSERT(string::npos != openParen); + // TODO: move forward until parens are balanced and terminator =,) + TextParser params("", &fName[openParen] + 1, &*fName.end(), 0); + bool underline = false; + while (!params.eof()) { +// SkDEBUGCODE(const char* end = params.anyOf("(),=")); // unused for now +// SkASSERT(end[0] != '('); // fixme: put off handling nested parentheseses + if (params.startsWith("const") || params.startsWith("int") + || params.startsWith("Sk")) { + const char* wordStart = params.fChar; + params.skipToNonAlphaNum(); + if (underline) { + result += '_'; + } else { + underline = true; + } + result += string(wordStart, params.fChar - wordStart); + } else { + params.skipToNonAlphaNum(); + } + if (!params.eof() && '*' == params.peek()) { + if (underline) { + result += '_'; + } else { + underline = true; + } + result += "star"; + params.next(); + params.skipSpace(); + } + params.skipToAlpha(); + } + } + } + } + fFiddle = normalized_name(result); +} + +bool Definition::exampleToScript(string* result) const { + bool hasFiddle = true; + const Definition* platform = this->hasChild(MarkType::kPlatform); + if (platform) { + TextParser platParse(platform); + hasFiddle = !platParse.strnstr("!fiddle", platParse.fEnd); + } + if (!hasFiddle) { + *result = ""; + return true; + } + string text = this->extractText(Definition::TrimExtract::kNo); + const char drawWrapper[] = "void draw(SkCanvas* canvas) {"; + const char drawNoCanvas[] = "void draw(SkCanvas* ) {"; + size_t nonSpace = 0; + while (nonSpace < text.length() && ' ' >= text[nonSpace]) { + ++nonSpace; + } + bool hasFunc = !text.compare(nonSpace, sizeof(drawWrapper) - 1, drawWrapper); + bool noCanvas = !text.compare(nonSpace, sizeof(drawNoCanvas) - 1, drawNoCanvas); + bool hasCanvas = string::npos != text.find("SkCanvas canvas"); + SkASSERT(!hasFunc || !noCanvas); + bool textOut = string::npos != text.find("SkDebugf(") + || string::npos != text.find("dump(") + || string::npos != text.find("dumpHex("); + string heightStr = "256"; + string widthStr = "256"; + bool preprocessor = text[0] == '#'; + string normalizedName(fFiddle); + string code; + string imageStr = "0"; + for (auto const& iter : fChildren) { + switch (iter->fMarkType) { + case MarkType::kError: + result->clear(); + return true; + case MarkType::kHeight: + heightStr = string(iter->fContentStart, iter->fContentEnd - iter->fContentStart); + break; + case MarkType::kWidth: + widthStr = string(iter->fContentStart, iter->fContentEnd - iter->fContentStart); + break; + case MarkType::kDescription: + // ignore for now + break; + case MarkType::kFunction: { + // emit this, but don't wrap this in draw() + string funcText(iter->fContentStart, iter->fContentEnd - iter->fContentStart - 1); + size_t pos = 0; + while (pos < funcText.length() && ' ' > funcText[pos]) { + ++pos; + } + size_t indent = count_indent(funcText, pos, funcText.length()); + add_code(funcText, pos, funcText.length(), 0, indent, code); + code += "\\n"; + } break; + case MarkType::kComment: + break; + case MarkType::kImage: + imageStr = string(iter->fContentStart, iter->fContentEnd - iter->fContentStart); + break; + case MarkType::kToDo: + break; + case MarkType::kMarkChar: + case MarkType::kPlatform: + // ignore for now + break; + case MarkType::kStdOut: + textOut = true; + break; + default: + SkASSERT(0); // more coding to do + } + } + string textOutStr = textOut ? "true" : "false"; + size_t pos = 0; + while (pos < text.length() && ' ' > text[pos]) { + ++pos; + } + size_t end = text.length(); + size_t outIndent = 0; + size_t textIndent = count_indent(text, pos, end); + bool wrapCode = !hasFunc && !noCanvas && !preprocessor; + if (wrapCode) { + code += hasCanvas ? drawNoCanvas : drawWrapper; + code += "\\n"; + outIndent = 4; + } + add_code(text, pos, end, outIndent, textIndent, code); + if (wrapCode) { + code += "}"; + } + string example = "\"" + normalizedName + "\": {\n"; + example += " \"code\": \"" + code + "\",\n"; + example += " \"options\": {\n"; + example += " \"width\": " + widthStr + ",\n"; + example += " \"height\": " + heightStr + ",\n"; + example += " \"source\": " + imageStr + ",\n"; + example += " \"srgb\": false,\n"; + example += " \"f16\": false,\n"; + example += " \"textOnly\": " + textOutStr + ",\n"; + example += " \"animated\": false,\n"; + example += " \"duration\": 0\n"; + example += " },\n"; + example += " \"fast\": true\n"; + example += "}"; + *result = example; + return true; +} + +static void space_pad(string* str) { + size_t len = str->length(); + if (len == 0) { + return; + } + char last = (*str)[len - 1]; + if ('~' == last || ' ' >= last) { + return; + } + *str += ' '; +} + +//start here; +// see if it possible to abstract this a little bit so it can +// additionally be used to find params and return in method prototype that +// does not have corresponding doxygen comments +bool Definition::checkMethod() const { + SkASSERT(MarkType::kMethod == fMarkType); + // if method returns a value, look for a return child + // for each parameter, look for a corresponding child + const char* end = fContentStart; + while (end > fStart && ' ' >= end[-1]) { + --end; + } + TextParser methodParser(fFileName, fStart, end, fLineCount); + methodParser.skipWhiteSpace(); + SkASSERT(methodParser.startsWith("#Method")); + methodParser.skipName("#Method"); + methodParser.skipSpace(); + string name = this->methodName(); + if (MethodType::kNone == fMethodType && "()" == name.substr(name.length() - 2)) { + name = name.substr(0, name.length() - 2); + } + bool expectReturn = this->methodHasReturn(name, &methodParser); + bool foundReturn = false; + bool foundException = false; + for (auto& child : fChildren) { + foundException |= MarkType::kDeprecated == child->fMarkType + || MarkType::kExperimental == child->fMarkType; + if (MarkType::kReturn != child->fMarkType) { + if (MarkType::kParam == child->fMarkType) { + child->fVisited = false; + } + continue; + } + if (!expectReturn) { + return methodParser.reportError<bool>("no #Return expected"); + } + if (foundReturn) { + return methodParser.reportError<bool>("multiple #Return markers"); + } + foundReturn = true; + } + if (expectReturn && !foundReturn && !foundException) { + return methodParser.reportError<bool>("missing #Return marker"); + } + const char* paren = methodParser.strnchr('(', methodParser.fEnd); + if (!paren) { + return methodParser.reportError<bool>("missing #Method function definition"); + } + const char* nextEnd = paren; + do { + string paramName; + methodParser.fChar = nextEnd + 1; + methodParser.skipSpace(); + if (!this->nextMethodParam(&methodParser, &nextEnd, ¶mName)) { + continue; + } + bool foundParam = false; + for (auto& child : fChildren) { + if (MarkType::kParam != child->fMarkType) { + continue; + } + if (paramName != child->fName) { + continue; + } + if (child->fVisited) { + return methodParser.reportError<bool>("multiple #Method param with same name"); + } + child->fVisited = true; + if (foundParam) { + TextParser paramError(child); + return methodParser.reportError<bool>("multiple #Param with same name"); + } + foundParam = true; + + } + if (!foundParam && !foundException) { + return methodParser.reportError<bool>("no #Param found"); + } + if (')' == nextEnd[0]) { + break; + } + } while (')' != nextEnd[0]); + for (auto& child : fChildren) { + if (MarkType::kParam != child->fMarkType) { + continue; + } + if (!child->fVisited) { + TextParser paramError(child); + return paramError.reportError<bool>("#Param without param in #Method"); + } + } + return true; +} + +bool Definition::crossCheck(const char* tokenID, const Definition& includeToken) const { + const char* defStart = fStart; + SkASSERT('#' == defStart[0]); // FIXME: needs to be per definition + ++defStart; + SkASSERT(!strncmp(defStart, tokenID, strlen(tokenID))); + defStart += strlen(tokenID); + return crossCheckInside(defStart, fContentStart, includeToken); +} + +bool Definition::crossCheck(const Definition& includeToken) const { + return crossCheckInside(fContentStart, fContentEnd, includeToken); +} + +bool Definition::crossCheckInside(const char* start, const char* end, + const Definition& includeToken) const { + TextParser def(fFileName, start, end, fLineCount); + TextParser inc("", includeToken.fContentStart, includeToken.fContentEnd, 0); + if (inc.startsWith("SK_API")) { + inc.skipWord("SK_API"); + } + if (inc.startsWith("friend")) { + inc.skipWord("friend"); + } + do { + bool defEof; + bool incEof; + do { + defEof = def.eof() || !def.skipWhiteSpace(); + incEof = inc.eof() || !inc.skipWhiteSpace(); + if (!incEof && '/' == inc.peek() && (defEof || '/' != def.peek())) { + inc.next(); + if ('*' == inc.peek()) { + inc.skipToEndBracket("*/"); + inc.next(); + } else if ('/' == inc.peek()) { + inc.skipToEndBracket('\n'); + } + } else if (!incEof && '#' == inc.peek() && (defEof || '#' != def.peek())) { + inc.next(); + SkASSERT(inc.startsWith("if")); + inc.skipToEndBracket("#"); + SkASSERT(inc.startsWith("#endif")); + inc.skipToEndBracket("\n"); + } else { + break; + } + inc.next(); + } while (true); + if (defEof || incEof) { + return defEof == incEof || (!defEof && ';' == def.peek()); + } + char defCh; + do { + defCh = def.next(); + char incCh = inc.next(); + if (' ' >= defCh && ' ' >= incCh) { + break; + } + if (defCh != incCh) { + return false; + } + if (';' == defCh) { + return true; + } + } while (!def.eof() && !inc.eof()); + } while (true); + return false; +} + +string Definition::formatFunction() const { + const char* end = fContentStart; + while (end > fStart && ' ' >= end[-1]) { + --end; + } + TextParser methodParser(fFileName, fStart, end, fLineCount); + methodParser.skipWhiteSpace(); + SkASSERT(methodParser.startsWith("#Method")); + methodParser.skipName("#Method"); + methodParser.skipSpace(); + const char* lastStart = methodParser.fChar; + const int limit = 80; // todo: allow this to be set by caller or in global or something + string methodStr; + string name = this->methodName(); + const char* nameInParser = methodParser.strnstr(name.c_str(), methodParser.fEnd); + methodParser.skipTo(nameInParser); + const char* lastEnd = methodParser.fChar; + const char* paren = methodParser.strnchr('(', methodParser.fEnd); + size_t indent; + if (paren) { + indent = (size_t) (paren - lastStart) + 1; + } else { + indent = (size_t) (lastEnd - lastStart); + } + int written = 0; + do { + const char* nextStart = lastEnd; + SkASSERT(written < limit); + const char* delimiter = methodParser.anyOf(",)"); + const char* nextEnd = delimiter ? delimiter : methodParser.fEnd; + if (delimiter) { + while (nextStart < nextEnd && ' ' >= nextStart[0]) { + ++nextStart; + } + } + while (nextEnd > nextStart && ' ' >= nextEnd[-1]) { + --nextEnd; + } + if (delimiter) { + nextEnd += 1; + delimiter += 1; + } + if (lastEnd > lastStart) { + if (lastStart[0] != ' ') { + space_pad(&methodStr); + } + methodStr += string(lastStart, (size_t) (lastEnd - lastStart)); + written += (size_t) (lastEnd - lastStart); + } + if (delimiter) { + if (nextEnd - nextStart >= (ptrdiff_t) (limit - written)) { + written = indent; + methodStr += '\n'; + methodStr += string(indent, ' '); + } + methodParser.skipTo(delimiter); + } + lastStart = nextStart; + lastEnd = nextEnd; + } while (lastStart < lastEnd); + return methodStr; +} + +string Definition::fiddleName() const { + string result; + size_t start = 0; + string parent; + const Definition* parentDef = this; + while ((parentDef = parentDef->fParent)) { + if (MarkType::kClass == parentDef->fMarkType || MarkType::kStruct == parentDef->fMarkType) { + parent = parentDef->fFiddle; + break; + } + } + if (parent.length() && 0 == fFiddle.compare(0, parent.length(), parent)) { + start = parent.length(); + while (start < fFiddle.length() && '_' == fFiddle[start]) { + ++start; + } + } + size_t end = fFiddle.find_first_of('(', start); + return fFiddle.substr(start, end - start); +} + +const Definition* Definition::hasChild(MarkType markType) const { + for (auto iter : fChildren) { + if (markType == iter->fMarkType) { + return iter; + } + } + return nullptr; +} + +const Definition* Definition::hasParam(const string& ref) const { + SkASSERT(MarkType::kMethod == fMarkType); + for (auto iter : fChildren) { + if (MarkType::kParam != iter->fMarkType) { + continue; + } + if (iter->fName == ref) { + return &*iter; + } + + } + return nullptr; +} + +bool Definition::methodHasReturn(const string& name, TextParser* methodParser) const { + const char* lastStart = methodParser->fChar; + const char* nameInParser = methodParser->strnstr(name.c_str(), methodParser->fEnd); + methodParser->skipTo(nameInParser); + const char* lastEnd = methodParser->fChar; + const char* returnEnd = lastEnd; + while (returnEnd > lastStart && ' ' == returnEnd[-1]) { + --returnEnd; + } + bool expectReturn = 4 != returnEnd - lastStart || strncmp("void", lastStart, 4); + if (MethodType::kNone != fMethodType && !expectReturn) { + return methodParser->reportError<bool>("unexpected void"); + } + switch (fMethodType) { + case MethodType::kNone: + case MethodType::kOperator: + // either is fine + break; + case MethodType::kConstructor: + expectReturn = true; + break; + case MethodType::kDestructor: + expectReturn = false; + break; + } + return expectReturn; +} + +string Definition::methodName() const { + string result; + size_t start = 0; + string parent; + const Definition* parentDef = this; + while ((parentDef = parentDef->fParent)) { + if (MarkType::kClass == parentDef->fMarkType || MarkType::kStruct == parentDef->fMarkType) { + parent = parentDef->fName; + break; + } + } + if (parent.length() && 0 == fName.compare(0, parent.length(), parent)) { + start = parent.length(); + while (start < fName.length() && ':' == fName[start]) { + ++start; + } + } + if (fClone) { + int lastUnder = fName.rfind('_'); + return fName.substr(start, (size_t) (lastUnder - start)); + } + size_t end = fName.find_first_of('(', start); + if (string::npos == end) { + return fName.substr(start); + } + return fName.substr(start, end - start); +} + +bool Definition::nextMethodParam(TextParser* methodParser, const char** nextEndPtr, + string* paramName) const { + *nextEndPtr = methodParser->anyOf(",)"); + const char* nextEnd = *nextEndPtr; + if (!nextEnd) { + return methodParser->reportError<bool>("#Method function missing close paren"); + } + const char* paramEnd = nextEnd; + const char* assign = methodParser->strnstr(" = ", paramEnd); + if (assign) { + paramEnd = assign; + } + const char* closeBracket = methodParser->strnstr("]", paramEnd); + if (closeBracket) { + const char* openBracket = methodParser->strnstr("[", paramEnd); + if (openBracket && openBracket < closeBracket) { + while (openBracket < --closeBracket && isdigit(closeBracket[0])) + ; + if (openBracket == closeBracket) { + paramEnd = openBracket; + } + } + } + while (paramEnd > methodParser->fChar && ' ' == paramEnd[-1]) { + --paramEnd; + } + const char* paramStart = paramEnd; + while (paramStart > methodParser->fChar && isalnum(paramStart[-1])) { + --paramStart; + } + if (paramStart > methodParser->fChar && paramStart >= paramEnd) { + return methodParser->reportError<bool>("#Method missing param name"); + } + *paramName = string(paramStart, paramEnd - paramStart); + if (!paramName->length()) { + if (')' != nextEnd[0]) { + return methodParser->reportError<bool>("#Method malformed param"); + } + return false; + } + return true; +} + + bool ParserCommon::parseFile(const char* fileOrPath, const char* suffix) { + if (!sk_isdir(fileOrPath)) { + if (!this->parseFromFile(fileOrPath)) { + SkDebugf("failed to parse %s\n", fileOrPath); + return false; + } + } else { + SkOSFile::Iter it(fileOrPath, suffix); + for (SkString file; it.next(&file); ) { + SkString p = SkOSPath::Join(fileOrPath, file.c_str()); + const char* hunk = p.c_str(); + if (!SkStrEndsWith(hunk, suffix)) { + continue; + } + if (!this->parseFromFile(hunk)) { + SkDebugf("failed to parse %s\n", hunk); + return false; + } + } + } + return true; +} + +bool Definition::paramsMatch(const string& match, const string& name) const { + TextParser def(fFileName, fStart, fContentStart, fLineCount); + const char* dName = def.strnstr(name.c_str(), fContentStart); + if (!dName) { + return false; + } + def.skipTo(dName); + TextParser m(fFileName, &match.front(), &match.back() + 1, fLineCount); + const char* mName = m.strnstr(name.c_str(), m.fEnd); + if (!mName) { + return false; + } + m.skipTo(mName); + while (!def.eof() && ')' != def.peek() && !m.eof() && ')' != m.peek()) { + const char* ds = def.fChar; + const char* ms = m.fChar; + const char* de = def.anyOf(") \n"); + const char* me = m.anyOf(") \n"); + def.skipTo(de); + m.skipTo(me); + if (def.fChar - ds != m.fChar - ms) { + return false; + } + if (strncmp(ds, ms, (int) (def.fChar - ds))) { + return false; + } + def.skipWhiteSpace(); + m.skipWhiteSpace(); + } + return !def.eof() && ')' == def.peek() && !m.eof() && ')' == m.peek(); +} + +void RootDefinition::clearVisited() { + fVisited = false; + for (auto& leaf : fLeaves) { + leaf.second.fVisited = false; + } + for (auto& branch : fBranches) { + branch.second->clearVisited(); + } +} + +bool RootDefinition::dumpUnVisited() { + bool allStructElementsFound = true; + for (auto& leaf : fLeaves) { + if (!leaf.second.fVisited) { + // TODO: parse embedded struct in includeParser phase, then remove this condition + size_t firstColon = leaf.first.find("::"); + size_t lastColon = leaf.first.rfind("::"); + if (firstColon != lastColon) { // struct, two sets + allStructElementsFound = false; + continue; + } + SkDebugf("defined in bmh but missing in include: %s\n", leaf.first.c_str()); + } + } + for (auto& branch : fBranches) { + allStructElementsFound &= branch.second->dumpUnVisited(); + } + return allStructElementsFound; +} + +const Definition* RootDefinition::find(const string& ref) const { + const auto leafIter = fLeaves.find(ref); + if (leafIter != fLeaves.end()) { + return &leafIter->second; + } + const auto branchIter = fBranches.find(ref); + if (branchIter != fBranches.end()) { + const RootDefinition* rootDef = branchIter->second; + return rootDef; + } + const Definition* result = nullptr; + for (const auto& branch : fBranches) { + const RootDefinition* rootDef = branch.second; + result = rootDef->find(ref); + if (result) { + break; + } + } + return result; +} + +/* + class contains named struct, enum, enum-member, method, topic, subtopic + everything contained by class is uniquely named + contained names may be reused by other classes + method contains named parameters + parameters may be reused in other methods + */ + +bool BmhParser::addDefinition(const char* defStart, bool hasEnd, MarkType markType, + const vector<string>& typeNameBuilder) { + Definition* definition = nullptr; + switch (markType) { + case MarkType::kComment: + if (!this->skipToDefinitionEnd(markType)) { + return false; + } + return true; + // these types may be referred to by name + case MarkType::kClass: + case MarkType::kStruct: + case MarkType::kConst: + case MarkType::kEnum: + case MarkType::kEnumClass: + case MarkType::kMember: + case MarkType::kMethod: + case MarkType::kTypedef: { + if (!typeNameBuilder.size()) { + return this->reportError<bool>("unnamed markup"); + } + if (typeNameBuilder.size() > 1) { + return this->reportError<bool>("expected one name only"); + } + const string& name = typeNameBuilder[0]; + if (nullptr == fRoot) { + fRoot = this->findBmhObject(markType, name); + fRoot->fFileName = fFileName; + definition = fRoot; + } else { + if (nullptr == fParent) { + return this->reportError<bool>("expected parent"); + } + if (fParent == fRoot && hasEnd) { + RootDefinition* rootParent = fRoot->rootParent(); + if (rootParent) { + fRoot = rootParent; + } + definition = fParent; + } else { + if (!hasEnd && fRoot->find(name)) { + return this->reportError<bool>("duplicate symbol"); + } + if (MarkType::kStruct == markType || MarkType::kClass == markType) { + // if class or struct, build fRoot hierarchy + // and change isDefined to search all parents of fRoot + SkASSERT(!hasEnd); + RootDefinition* childRoot = new RootDefinition; + (fRoot->fBranches)[name] = childRoot; + childRoot->setRootParent(fRoot); + childRoot->fFileName = fFileName; + fRoot = childRoot; + definition = fRoot; + } else { + definition = &fRoot->fLeaves[name]; + } + } + } + if (hasEnd) { + Exemplary hasExample = Exemplary::kNo; + bool hasExcluder = false; + for (auto child : definition->fChildren) { + if (MarkType::kExample == child->fMarkType) { + hasExample = Exemplary::kYes; + } + hasExcluder |= MarkType::kPrivate == child->fMarkType + || MarkType::kDeprecated == child->fMarkType + || MarkType::kExperimental == child->fMarkType + || MarkType::kNoExample == child->fMarkType; + } + if (fMaps[(int) markType].fExemplary != hasExample + && fMaps[(int) markType].fExemplary != Exemplary::kOptional) { + if (string::npos == fFileName.find("undocumented") + && !hasExcluder) { + hasExample == Exemplary::kNo ? + this->reportWarning("missing example") : + this->reportWarning("unexpected example"); + } + + } + if (MarkType::kMethod == markType) { + if (fCheckMethods && !definition->checkMethod()) { + return false; + } + } + if (!this->popParentStack(definition)) { + return false; + } + } else { + definition->fStart = defStart; + this->skipSpace(); + definition->fFileName = fFileName; + definition->fContentStart = fChar; + definition->fLineCount = fLineCount; + definition->fClone = fCloned; + if (MarkType::kConst == markType) { + // todo: require that fChar points to def on same line as markup + // additionally add definition to class children if it is not already there + if (definition->fParent != fRoot) { +// fRoot->fChildren.push_back(definition); + } + } + definition->fName = name; + if (MarkType::kMethod == markType) { + if (string::npos != name.find(':', 0)) { + definition->setCanonicalFiddle(); + } else { + definition->fFiddle = name; + } + } else { + definition->fFiddle = normalized_name(name); + } + definition->fMarkType = markType; + this->setAsParent(definition); + } + } break; + case MarkType::kTopic: + case MarkType::kSubtopic: + SkASSERT(1 == typeNameBuilder.size()); + if (!hasEnd) { + if (!typeNameBuilder.size()) { + return this->reportError<bool>("unnamed topic"); + } + fTopics.emplace_front(markType, defStart, fLineCount, fParent); + RootDefinition* rootDefinition = &fTopics.front(); + definition = rootDefinition; + definition->fFileName = fFileName; + definition->fContentStart = fChar; + definition->fName = typeNameBuilder[0]; + Definition* parent = fParent; + while (parent && MarkType::kTopic != parent->fMarkType + && MarkType::kSubtopic != parent->fMarkType) { + parent = parent->fParent; + } + definition->fFiddle = parent ? parent->fFiddle + '_' : ""; + definition->fFiddle += normalized_name(typeNameBuilder[0]); + this->setAsParent(definition); + } + { + const string& fullTopic = hasEnd ? fParent->fFiddle : definition->fFiddle; + Definition* defPtr = fTopicMap[fullTopic]; + if (hasEnd) { + if (!definition) { + definition = defPtr; + } else if (definition != defPtr) { + return this->reportError<bool>("mismatched topic"); + } + } else { + if (nullptr != defPtr) { + return this->reportError<bool>("already declared topic"); + } + fTopicMap[fullTopic] = definition; + } + } + if (hasEnd) { + if (!this->popParentStack(definition)) { + return false; + } + } + break; + // these types are children of parents, but are not in named maps + case MarkType::kDefinedBy: { + string prefixed(fRoot->fName); + const char* start = fChar; + string name(start, this->trimmedBracketEnd(fMC, OneLine::kYes) - start); + prefixed += "::" + name; + this->skipToEndBracket(fMC); + const auto leafIter = fRoot->fLeaves.find(prefixed); + if (fRoot->fLeaves.end() != leafIter) { + this->reportError<bool>("DefinedBy already defined"); + } + definition = &fRoot->fLeaves[prefixed]; + definition->fParent = fParent; + definition->fStart = defStart; + definition->fContentStart = start; + definition->fName = name; + definition->fFiddle = normalized_name(name); + definition->fContentEnd = fChar; + this->skipToEndBracket('\n'); + definition->fTerminator = fChar; + definition->fMarkType = markType; + definition->fLineCount = fLineCount; + fParent->fChildren.push_back(definition); + } break; + case MarkType::kDescription: + case MarkType::kStdOut: + // may be one-liner + case MarkType::kBug: + case MarkType::kNoExample: + case MarkType::kParam: + case MarkType::kReturn: + case MarkType::kToDo: + if (hasEnd) { + if (markType == fParent->fMarkType) { + definition = fParent; + if (MarkType::kBug == markType || MarkType::kReturn == markType + || MarkType::kToDo == markType) { + this->skipNoName(); + } + if (!this->popParentStack(fParent)) { // if not one liner, pop + return false; + } + } else { + fMarkup.emplace_front(markType, defStart, fLineCount, fParent); + definition = &fMarkup.front(); + definition->fName = typeNameBuilder[0]; + definition->fFiddle = normalized_name(typeNameBuilder[0]); + definition->fContentStart = fChar; + definition->fContentEnd = this->trimmedBracketEnd(fMC, OneLine::kYes); + this->skipToEndBracket(fMC); + SkAssertResult(fMC == this->next()); + SkAssertResult(fMC == this->next()); + definition->fTerminator = fChar; + fParent->fChildren.push_back(definition); + } + break; + } + // not one-liners + case MarkType::kCode: + case MarkType::kDeprecated: + case MarkType::kExample: + case MarkType::kExperimental: + case MarkType::kFormula: + case MarkType::kFunction: + case MarkType::kLegend: + case MarkType::kList: + case MarkType::kPrivate: + case MarkType::kTable: + case MarkType::kTrack: + if (hasEnd) { + definition = fParent; + if (markType != fParent->fMarkType) { + return this->reportError<bool>("end element mismatch"); + } else if (!this->popParentStack(fParent)) { + return false; + } + if (MarkType::kExample == markType) { + if (definition->fChildren.size() == 0) { + TextParser emptyCheck(definition); + if (emptyCheck.eof() || !emptyCheck.skipWhiteSpace()) { + return this->reportError<bool>("missing example body"); + } + } + } + } else { + fMarkup.emplace_front(markType, defStart, fLineCount, fParent); + definition = &fMarkup.front(); + definition->fContentStart = fChar; + definition->fName = typeNameBuilder[0]; + definition->fFiddle = fParent->fFiddle; + char suffix = '\0'; + bool tryAgain; + do { + tryAgain = false; + for (const auto& child : fParent->fChildren) { + if (child->fFiddle == definition->fFiddle) { + if (MarkType::kExample != child->fMarkType) { + continue; + } + if ('\0' == suffix) { + suffix = 'a'; + } else if (++suffix > 'z') { + return reportError<bool>("too many examples"); + } + definition->fFiddle = fParent->fFiddle + '_'; + definition->fFiddle += suffix; + tryAgain = true; + break; + } + } + } while (tryAgain); + this->setAsParent(definition); + } + break; + // always treated as one-liners (can't detect misuse easily) + case MarkType::kAlias: + case MarkType::kAnchor: + case MarkType::kDefine: + case MarkType::kError: + case MarkType::kFile: + case MarkType::kHeight: + case MarkType::kImage: + case MarkType::kPlatform: + case MarkType::kSeeAlso: + case MarkType::kSubstitute: + case MarkType::kTime: + case MarkType::kVolatile: + case MarkType::kWidth: + if (hasEnd) { + return this->reportError<bool>("one liners omit end element"); + } + fMarkup.emplace_front(markType, defStart, fLineCount, fParent); + definition = &fMarkup.front(); + definition->fName = typeNameBuilder[0]; + definition->fFiddle = normalized_name(typeNameBuilder[0]); + definition->fContentStart = fChar; + definition->fContentEnd = this->trimmedBracketEnd('\n', OneLine::kYes); + definition->fTerminator = this->lineEnd() - 1; + fParent->fChildren.push_back(definition); + if (MarkType::kAnchor == markType) { + this->skipToEndBracket(fMC); + fMarkup.emplace_front(MarkType::kLink, fChar, fLineCount, definition); + SkAssertResult(fMC == this->next()); + this->skipWhiteSpace(); + Definition* link = &fMarkup.front(); + link->fContentStart = fChar; + link->fContentEnd = this->trimmedBracketEnd(fMC, OneLine::kYes); + this->skipToEndBracket(fMC); + SkAssertResult(fMC == this->next()); + SkAssertResult(fMC == this->next()); + link->fTerminator = fChar; + definition->fContentEnd = link->fContentEnd; + definition->fTerminator = fChar; + definition->fChildren.emplace_back(link); + } else if (MarkType::kAlias == markType) { + this->skipWhiteSpace(); + const char* start = fChar; + this->skipToNonAlphaNum(); + string alias(start, fChar - start); + if (fAliasMap.end() != fAliasMap.find(alias)) { + return this->reportError<bool>("duplicate alias"); + } + fAliasMap[alias] = definition; + } + break; + case MarkType::kExternal: + (void) this->collectExternals(); // FIXME: detect errors in external defs? + break; + default: + SkASSERT(0); // fixme : don't let any types be invisible + return true; + } + if (fParent) { + SkASSERT(definition); + SkASSERT(definition->fName.length() > 0); + } + return true; +} + +bool BmhParser::childOf(MarkType markType) const { + auto childError = [this](MarkType markType) -> bool { + string errStr = "expected "; + errStr += fMaps[(int) markType].fName; + errStr += " parent"; + return this->reportError<bool>(errStr.c_str()); + }; + + if (markType == fParent->fMarkType) { + return true; + } + if (this->hasEndToken()) { + if (!fParent->fParent) { + return this->reportError<bool>("expected grandparent"); + } + if (markType == fParent->fParent->fMarkType) { + return true; + } + } + return childError(markType); +} + +string BmhParser::className(MarkType markType) { + string builder; + const Definition* parent = this->parentSpace(); + if (parent && (parent != fParent || MarkType::kClass != markType)) { + builder += parent->fName; + } + const char* end = this->lineEnd(); + const char* mc = this->strnchr(fMC, end); + if (mc) { + this->skipSpace(); + const char* wordStart = fChar; + this->skipToNonAlphaNum(); + const char* wordEnd = fChar; + if (mc + 1 < fEnd && fMC == mc[1]) { // if ## + if (markType != fParent->fMarkType) { + return this->reportError<string>("unbalanced method"); + } + if (builder.length() > 0 && wordEnd > wordStart) { + if (builder != fParent->fName) { + builder += "::"; + builder += string(wordStart, wordEnd - wordStart); + if (builder != fParent->fName) { + return this->reportError<string>("name mismatch"); + } + } + } + this->skipLine(); + return fParent->fName; + } + fChar = mc; + this->next(); + } + this->skipWhiteSpace(); + if (MarkType::kEnum == markType && fChar >= end) { + fAnonymous = true; + builder += "::_anonymous"; + return uniqueRootName(builder, markType); + } + builder = this->word(builder, "::"); + return builder; +} + +bool BmhParser::collectExternals() { + do { + this->skipWhiteSpace(); + if (this->eof()) { + break; + } + if (fMC == this->peek()) { + this->next(); + if (this->eof()) { + break; + } + if (fMC == this->peek()) { + this->skipLine(); + break; + } + if (' ' >= this->peek()) { + this->skipLine(); + continue; + } + if (this->startsWith(fMaps[(int) MarkType::kExternal].fName)) { + this->skipToNonAlphaNum(); + continue; + } + } + this->skipToAlpha(); + const char* wordStart = fChar; + this->skipToNonAlphaNum(); + if (fChar - wordStart > 0) { + fExternals.emplace_front(MarkType::kExternal, wordStart, fChar, fLineCount, fParent); + RootDefinition* definition = &fExternals.front(); + definition->fFileName = fFileName; + definition->fName = string(wordStart ,fChar - wordStart); + definition->fFiddle = normalized_name(definition->fName); + } + } while (!this->eof()); + return true; +} + +int BmhParser::endHashCount() const { + const char* end = fLine + this->lineLength(); + int count = 0; + while (fLine < end && fMC == *--end) { + count++; + } + return count; +} + +// FIXME: some examples may produce different output on different platforms +// if the text output can be different, think of how to author that + +bool BmhParser::findDefinitions() { + bool lineStart = true; + fParent = nullptr; + while (!this->eof()) { + if (this->peek() == fMC) { + this->next(); + if (this->peek() == fMC) { + this->next(); + if (!lineStart && ' ' < this->peek()) { + return this->reportError<bool>("expected definition"); + } + if (this->peek() != fMC) { + vector<string> parentName; + parentName.push_back(fParent->fName); + if (!this->addDefinition(fChar - 1, true, fParent->fMarkType, parentName)) { + return false; + } + } else { + SkAssertResult(this->next() == fMC); + fMC = this->next(); // change markup character + if (' ' >= fMC) { + return this->reportError<bool>("illegal markup character"); + } + fMarkup.emplace_front(MarkType::kMarkChar, fChar - 1, fLineCount, fParent); + Definition* markChar = &fMarkup.front(); + markChar->fContentStart = fChar - 1; + this->skipToEndBracket('\n'); + markChar->fContentEnd = fChar; + markChar->fTerminator = fChar; + fParent->fChildren.push_back(markChar); + } + } else if (this->peek() >= 'A' && this->peek() <= 'Z') { + const char* defStart = fChar - 1; + MarkType markType = this->getMarkType(MarkLookup::kRequire); + bool hasEnd = this->hasEndToken(); + if (!hasEnd) { + MarkType parentType = fParent ? fParent->fMarkType : MarkType::kRoot; + uint64_t parentMask = fMaps[(int) markType].fParentMask; + if (parentMask && !(parentMask & (1LL << (int) parentType))) { + return this->reportError<bool>("invalid parent"); + } + } + if (!this->skipName(fMaps[(int) markType].fName)) { + return this->reportError<bool>("illegal markup character"); + } + if (!this->skipSpace()) { + return this->reportError<bool>("unexpected end"); + } + bool expectEnd = true; + vector<string> typeNameBuilder = this->typeName(markType, &expectEnd); + if (fCloned && MarkType::kMethod != markType && MarkType::kExample != markType + && !fAnonymous) { + return this->reportError<bool>("duplicate name"); + } + if (hasEnd && expectEnd) { + SkASSERT(fMC != this->peek()); + } + if (!this->addDefinition(defStart, hasEnd, markType, typeNameBuilder)) { + return false; + } + continue; + } else if (this->peek() == ' ') { + if (!fParent || (MarkType::kTable != fParent->fMarkType + && MarkType::kLegend != fParent->fMarkType + && MarkType::kList != fParent->fMarkType)) { + int endHashes = this->endHashCount(); + if (endHashes <= 1) { // one line comment + if (fParent) { + fMarkup.emplace_front(MarkType::kComment, fChar - 1, fLineCount, fParent); + Definition* comment = &fMarkup.front(); + comment->fContentStart = fChar - 1; + this->skipToEndBracket('\n'); + comment->fContentEnd = fChar; + comment->fTerminator = fChar; + fParent->fChildren.push_back(comment); + } else { + fChar = fLine + this->lineLength() - 1; + } + } else { // table row + if (2 != endHashes) { + string errorStr = "expect "; + errorStr += fMC; + errorStr += fMC; + return this->reportError<bool>(errorStr.c_str()); + } + if (!fParent || MarkType::kTable != fParent->fMarkType) { + return this->reportError<bool>("missing table"); + } + } + } else { + bool parentIsList = MarkType::kList == fParent->fMarkType; + // fixme? no nested tables for now + const char* colStart = fChar - 1; + fMarkup.emplace_front(MarkType::kRow, colStart, fLineCount, fParent); + Definition* row = &fMarkup.front(); + this->skipWhiteSpace(); + row->fContentStart = fChar; + this->setAsParent(row); + const char* lineEnd = this->lineEnd(); + do { + fMarkup.emplace_front(MarkType::kColumn, colStart, fLineCount, fParent); + Definition* column = &fMarkup.front(); + column->fContentStart = fChar; + column->fContentEnd = this->trimmedBracketEnd(fMC, + parentIsList ? OneLine::kNo : OneLine::kYes); + this->skipToEndBracket(fMC); + colStart = fChar; + SkAssertResult(fMC == this->next()); + if (fMC == this->peek()) { + this->next(); + } + column->fTerminator = fChar; + fParent->fChildren.push_back(column); + this->skipSpace(); + } while (fChar < lineEnd && '\n' != this->peek()); + if (!this->popParentStack(fParent)) { + return false; + } + const Definition* lastCol = row->fChildren.back(); + row->fContentEnd = lastCol->fContentEnd; + } + } + } + lineStart = this->next() == '\n'; + } + if (fParent) { + return this->reportError<bool>("mismatched end"); + } + return true; +} + +MarkType BmhParser::getMarkType(MarkLookup lookup) const { + for (int index = 0; index <= Last_MarkType; ++index) { + int typeLen = strlen(fMaps[index].fName); + if (typeLen == 0) { + continue; + } + if (fChar + typeLen >= fEnd || fChar[typeLen] > ' ') { + continue; + } + int chCompare = strncmp(fChar, fMaps[index].fName, typeLen); + if (chCompare < 0) { + goto fail; + } + if (chCompare == 0) { + return (MarkType) index; + } + } +fail: + if (MarkLookup::kRequire == lookup) { + return this->reportError<MarkType>("unknown mark type"); + } + return MarkType::kNone; +} + +bool HackParser::hackFiles() { + string filename(fFileName); + size_t len = filename.length() - 1; + while (len > 0 && (isalnum(filename[len]) || '_' == filename[len] || '.' == filename[len])) { + --len; + } + filename = filename.substr(len + 1); + // remove trailing period from #Param and #Return + FILE* out = fopen(filename.c_str(), "wb"); + if (!out) { + SkDebugf("could not open output file %s\n", filename.c_str()); + return false; + } + const char* start = fStart; + do { + const char* match = this->strnchr('#', fEnd); + if (!match) { + break; + } + this->skipTo(match); + this->next(); + if (!this->startsWith("Param") && !this->startsWith("Return")) { + continue; + } + const char* end = this->strnstr("##", fEnd); + while (true) { + TextParser::Save lastPeriod(this); + this->next(); + if (!this->skipToEndBracket('.', end)) { + lastPeriod.restore(); + break; + } + } + if ('.' == this->peek()) { + fprintf(out, "%.*s", (int) (fChar - start), start); + this->next(); + start = fChar; + } + } while (!this->eof()); + fprintf(out, "%.*s", (int) (fEnd - start), start); + fclose(out); + return true; +} + +bool BmhParser::hasEndToken() const { + const char* last = fLine + this->lineLength(); + while (last > fLine && ' ' >= *--last) + ; + if (--last < fLine) { + return false; + } + return last[0] == fMC && last[1] == fMC; +} + +string BmhParser::memberName() { + const char* wordStart; + const char* prefixes[] = { "static", "const" }; + do { + this->skipSpace(); + wordStart = fChar; + this->skipToNonAlphaNum(); + } while (this->anyOf(wordStart, prefixes, SK_ARRAY_COUNT(prefixes))); + if ('*' == this->peek()) { + this->next(); + } + return this->className(MarkType::kMember); +} + +string BmhParser::methodName() { + if (this->hasEndToken()) { + if (!fParent || !fParent->fName.length()) { + return this->reportError<string>("missing parent method name"); + } + SkASSERT(fMC == this->peek()); + this->next(); + SkASSERT(fMC == this->peek()); + this->next(); + SkASSERT(fMC != this->peek()); + return fParent->fName; + } + string builder; + const char* end = this->lineEnd(); + const char* paren = this->strnchr('(', end); + if (!paren) { + return this->reportError<string>("missing method name and reference"); + } + const char* nameStart = paren; + char ch; + bool expectOperator = false; + bool isConstructor = false; + const char* nameEnd = nullptr; + while (nameStart > fChar && ' ' != (ch = *--nameStart)) { + if (!isalnum(ch) && '_' != ch) { + if (nameEnd) { + break; + } + expectOperator = true; + continue; + } + if (!nameEnd) { + nameEnd = nameStart + 1; + } + } + if (!nameEnd) { + return this->reportError<string>("unexpected method name char"); + } + if (' ' == nameStart[0]) { + ++nameStart; + } + if (nameEnd <= nameStart) { + return this->reportError<string>("missing method name"); + } + if (nameStart >= paren) { + return this->reportError<string>("missing method name length"); + } + string name(nameStart, nameEnd - nameStart); + bool allLower = true; + for (int index = 0; index < (int) (nameEnd - nameStart); ++index) { + if (!islower(nameStart[index])) { + allLower = false; + break; + } + } + if (expectOperator && "operator" != name) { + return this->reportError<string>("expected operator"); + } + const Definition* parent = this->parentSpace(); + if (parent && parent->fName.length() > 0) { + if (parent->fName == name) { + isConstructor = true; + } else if ('~' == name[0]) { + if (parent->fName != name.substr(1)) { + return this->reportError<string>("expected destructor"); + } + isConstructor = true; + } + builder = parent->fName + "::"; + } + if (isConstructor || expectOperator) { + paren = this->strnchr(')', end) + 1; + } + builder.append(nameStart, paren - nameStart); + if (!expectOperator && allLower) { + builder.append("()"); + } + int parens = 0; + while (fChar < end || parens > 0) { + if ('(' == this->peek()) { + ++parens; + } else if (')' == this->peek()) { + --parens; + } + this->next(); + } + TextParser::Save saveState(this); + this->skipWhiteSpace(); + if (this->startsWith("const")) { + this->skipName("const"); + } else { + saveState.restore(); + } +// this->next(); + return uniqueRootName(builder, MarkType::kMethod); +} + +const Definition* BmhParser::parentSpace() const { + Definition* parent = nullptr; + Definition* test = fParent; + while (test) { + if (MarkType::kClass == test->fMarkType || + MarkType::kEnumClass == test->fMarkType || + MarkType::kStruct == test->fMarkType) { + parent = test; + break; + } + test = test->fParent; + } + return parent; +} + +bool BmhParser::popParentStack(Definition* definition) { + if (!fParent) { + return this->reportError<bool>("missing parent"); + } + if (definition != fParent) { + return this->reportError<bool>("definition end is not parent"); + } + if (!definition->fStart) { + return this->reportError<bool>("definition missing start"); + } + if (definition->fContentEnd) { + return this->reportError<bool>("definition already ended"); + } + definition->fContentEnd = fLine - 1; + definition->fTerminator = fChar; + fParent = definition->fParent; + if (!fParent || (MarkType::kTopic == fParent->fMarkType && !fParent->fParent)) { + fRoot = nullptr; + } + return true; +} + +TextParser::TextParser(const Definition* definition) : + TextParser(definition->fFileName, definition->fContentStart, definition->fContentEnd, + definition->fLineCount) { +} + +void TextParser::reportError(const char* errorStr) const { + this->reportWarning(errorStr); + SkDebugf(""); // convenient place to set a breakpoint +} + +void TextParser::reportWarning(const char* errorStr) const { + TextParser err(fFileName, fLine, fEnd, fLineCount); + size_t lineLen = this->lineLength(); + ptrdiff_t spaces = fChar - fLine; + while (spaces > 0 && (size_t) spaces > lineLen) { + ++err.fLineCount; + err.fLine += lineLen; + spaces -= lineLen; + lineLen = err.lineLength(); + } + SkDebugf("%s(%zd): error: %s\n", fFileName.c_str(), err.fLineCount, errorStr); + if (0 == lineLen) { + SkDebugf("[blank line]\n"); + } else { + while (lineLen > 0 && '\n' == err.fLine[lineLen - 1]) { + --lineLen; + } + SkDebugf("%.*s\n", (int) lineLen, err.fLine); + SkDebugf("%*s^\n", (int) spaces, ""); + } +} + +bool BmhParser::skipNoName() { + if ('\n' == this->peek()) { + this->next(); + return true; + } + this->skipWhiteSpace(); + if (fMC != this->peek()) { + return this->reportError<bool>("expected end mark"); + } + this->next(); + if (fMC != this->peek()) { + return this->reportError<bool>("expected end mark"); + } + this->next(); + return true; +} + +bool BmhParser::skipToDefinitionEnd(MarkType markType) { + if (this->eof()) { + return this->reportError<bool>("missing end"); + } + const char* start = fLine; + int startLineCount = fLineCount; + int stack = 1; + ptrdiff_t lineLen; + bool foundEnd = false; + do { + lineLen = this->lineLength(); + if (fMC != *fChar++) { + continue; + } + if (fMC == *fChar) { + continue; + } + if (' ' == *fChar) { + continue; + } + MarkType nextType = this->getMarkType(MarkLookup::kAllowUnknown); + if (markType != nextType) { + continue; + } + bool hasEnd = this->hasEndToken(); + if (hasEnd) { + if (!--stack) { + foundEnd = true; + continue; + } + } else { + ++stack; + } + } while ((void) ++fLineCount, (void) (fLine += lineLen), (void) (fChar = fLine), + !this->eof() && !foundEnd); + if (foundEnd) { + return true; + } + fLineCount = startLineCount; + fLine = start; + fChar = start; + return this->reportError<bool>("unbalanced stack"); +} + +vector<string> BmhParser::topicName() { + vector<string> result; + this->skipWhiteSpace(); + const char* lineEnd = fLine + this->lineLength(); + const char* nameStart = fChar; + while (fChar < lineEnd) { + char ch = this->next(); + SkASSERT(',' != ch); + if ('\n' == ch) { + break; + } + if (fMC == ch) { + break; + } + } + if (fChar - 1 > nameStart) { + string builder(nameStart, fChar - nameStart - 1); + trim_start_end(builder); + result.push_back(builder); + } + if (fChar < lineEnd && fMC == this->peek()) { + this->next(); + } + return result; +} + +// typeName parsing rules depend on mark type +vector<string> BmhParser::typeName(MarkType markType, bool* checkEnd) { + fAnonymous = false; + fCloned = false; + vector<string> result; + string builder; + if (fParent) { + builder = fParent->fName; + } + switch (markType) { + case MarkType::kEnum: + // enums may be nameless + case MarkType::kConst: + case MarkType::kEnumClass: + case MarkType::kClass: + case MarkType::kStruct: + case MarkType::kTypedef: + // expect name + builder = this->className(markType); + break; + case MarkType::kExample: + // check to see if one already exists -- if so, number this one + builder = this->uniqueName(string(), markType); + this->skipNoName(); + break; + case MarkType::kCode: + case MarkType::kDeprecated: + case MarkType::kDescription: + case MarkType::kDoxygen: + case MarkType::kExperimental: + case MarkType::kExternal: + case MarkType::kFormula: + case MarkType::kFunction: + case MarkType::kLegend: + case MarkType::kList: + case MarkType::kNoExample: + case MarkType::kPrivate: + case MarkType::kTrack: + this->skipNoName(); + break; + case MarkType::kAlias: + case MarkType::kAnchor: + case MarkType::kBug: // fixme: expect number + case MarkType::kDefine: + case MarkType::kDefinedBy: + case MarkType::kError: + case MarkType::kFile: + case MarkType::kHeight: + case MarkType::kImage: + case MarkType::kPlatform: + case MarkType::kReturn: + case MarkType::kSeeAlso: + case MarkType::kSubstitute: + case MarkType::kTime: + case MarkType::kToDo: + case MarkType::kVolatile: + case MarkType::kWidth: + *checkEnd = false; // no name, may have text body + break; + case MarkType::kStdOut: + this->skipNoName(); + break; // unnamed + case MarkType::kMember: + builder = this->memberName(); + break; + case MarkType::kMethod: + builder = this->methodName(); + break; + case MarkType::kParam: + // fixme: expect camelCase + builder = this->word("", ""); + this->skipSpace(); + *checkEnd = false; + break; + case MarkType::kTable: + this->skipNoName(); + break; // unnamed + case MarkType::kSubtopic: + case MarkType::kTopic: + // fixme: start with cap, allow space, hyphen, stop on comma + // one topic can have multiple type names delineated by comma + result = this->topicName(); + if (result.size() == 0 && this->hasEndToken()) { + break; + } + return result; + default: + // fixme: don't allow silent failures + SkASSERT(0); + } + result.push_back(builder); + return result; +} + +string BmhParser::uniqueName(const string& base, MarkType markType) { + string builder(base); + if (!builder.length()) { + builder = fParent->fName; + } + if (!fParent) { + return builder; + } + int number = 2; + string numBuilder(builder); + do { + for (const auto& iter : fParent->fChildren) { + if (markType == iter->fMarkType) { + if (iter->fName == numBuilder) { + if (MarkType::kMethod == markType) { + SkDebugf(""); + } + fCloned = true; + numBuilder = builder + '_' + to_string(number); + goto tryNext; + } + } + } + break; +tryNext: ; + } while (++number); + return numBuilder; +} + +string BmhParser::uniqueRootName(const string& base, MarkType markType) { + auto checkName = [markType](const Definition& def, const string& numBuilder) -> bool { + return markType == def.fMarkType && def.fName == numBuilder; + }; + + string builder(base); + if (!builder.length()) { + builder = fParent->fName; + } + int number = 2; + string numBuilder(builder); + Definition* cloned = nullptr; + do { + if (fRoot) { + for (auto& iter : fRoot->fBranches) { + if (checkName(*iter.second, numBuilder)) { + cloned = iter.second; + goto tryNext; + } + } + for (auto& iter : fRoot->fLeaves) { + if (checkName(iter.second, numBuilder)) { + cloned = &iter.second; + goto tryNext; + } + } + } else if (fParent) { + for (auto& iter : fParent->fChildren) { + if (checkName(*iter, numBuilder)) { + cloned = &*iter; + goto tryNext; + } + } + } + break; +tryNext: ; + if ("()" == builder.substr(builder.length() - 2)) { + builder = builder.substr(0, builder.length() - 2); + } + if (MarkType::kMethod == markType) { + cloned->fCloned = true; + } + fCloned = true; + numBuilder = builder + '_' + to_string(number); + } while (++number); + return numBuilder; +} + +void BmhParser::validate() const { + for (int index = 0; index <= (int) Last_MarkType; ++index) { + SkASSERT(fMaps[index].fMarkType == (MarkType) index); + } + const char* last = ""; + for (int index = 0; index <= (int) Last_MarkType; ++index) { + const char* next = fMaps[index].fName; + if (!last[0]) { + last = next; + continue; + } + if (!next[0]) { + continue; + } + SkASSERT(strcmp(last, next) < 0); + last = next; + } +} + +string BmhParser::word(const string& prefix, const string& delimiter) { + string builder(prefix); + this->skipWhiteSpace(); + const char* lineEnd = fLine + this->lineLength(); + const char* nameStart = fChar; + while (fChar < lineEnd) { + char ch = this->next(); + if (' ' >= ch) { + break; + } + if (',' == ch) { + return this->reportError<string>("no comma needed"); + break; + } + if (fMC == ch) { + return builder; + } + if (!isalnum(ch) && '_' != ch && ':' != ch && '-' != ch) { + return this->reportError<string>("unexpected char"); + } + if (':' == ch) { + // expect pair, and expect word to start with Sk + if (nameStart[0] != 'S' || nameStart[1] != 'k') { + return this->reportError<string>("expected Sk"); + } + if (':' != this->peek()) { + return this->reportError<string>("expected ::"); + } + this->next(); + } else if ('-' == ch) { + // expect word not to start with Sk or kX where X is A-Z + if (nameStart[0] == 'k' && nameStart[1] >= 'A' && nameStart[1] <= 'Z') { + return this->reportError<string>("didn't expected kX"); + } + if (nameStart[0] == 'S' && nameStart[1] == 'k') { + return this->reportError<string>("expected Sk"); + } + } + } + if (prefix.size()) { + builder += delimiter; + } + builder.append(nameStart, fChar - nameStart - 1); + return builder; +} + +// pass one: parse text, collect definitions +// pass two: lookup references + +DEFINE_string2(bmh, b, "", "A path to a *.bmh file or a directory."); +DEFINE_string2(examples, e, "", "File of fiddlecli input, usually fiddle.json (For now, disables -r -f -s)"); +DEFINE_string2(fiddle, f, "fiddleout.json", "File of fiddlecli output."); +DEFINE_string2(include, i, "", "A path to a *.h file or a directory."); +DEFINE_bool2(hack, k, false, "Do a find/replace hack to update all *.bmh files. (Requires -b)"); +DEFINE_bool2(populate, p, false, "Populate include from bmh. (Requires -b -i)"); +DEFINE_string2(ref, r, "", "Resolve refs and write bmh_*.md files to path. (Requires -b)"); +DEFINE_bool2(spellcheck, s, false, "Spell-check. (Requires -b)"); +DEFINE_bool2(tokens, t, false, "Output include tokens. (Requires -i)"); +DEFINE_bool2(crosscheck, x, false, "Check bmh against includes. (Requires -b -i)"); + +static bool dump_examples(FILE* fiddleOut, const Definition& def, bool* continuation) { + if (MarkType::kExample == def.fMarkType) { + string result; + if (!def.exampleToScript(&result)) { + return false; + } + if (result.length() > 0) { + if (*continuation) { + fprintf(fiddleOut, ",\n"); + } else { + *continuation = true; + } + fprintf(fiddleOut, "%s", result.c_str()); + } + return true; + } + for (auto& child : def.fChildren ) { + if (!dump_examples(fiddleOut, *child, continuation)) { + return false; + } + } + return true; +} + +static int count_children(const Definition& def, MarkType markType) { + int count = 0; + if (markType == def.fMarkType) { + ++count; + } + for (auto& child : def.fChildren ) { + count += count_children(*child, markType); + } + return count; +} + +int main(int argc, char** const argv) { + BmhParser bmhParser; + bmhParser.validate(); + + SkCommandLineFlags::SetUsage( + "Common Usage: bookmaker -i path/to/include.h -t\n" + " bookmaker -b path/to/bmh_files -e fiddle.json\n" + " ~/go/bin/fiddlecli --input fiddle.json --output fiddleout.json\n" + " bookmaker -b path/to/bmh_files -f fiddleout.json -r path/to/md_files\n" + " bookmaker -b path/to/bmh_files -i path/to/include.h -x\n" + " bookmaker -b path/to/bmh_files -i path/to/include.h -p\n"); + bool help = false; + for (int i = 1; i < argc; i++) { + if (0 == strcmp("-h", argv[i]) || 0 == strcmp("--help", argv[i])) { + help = true; + for (int j = i + 1; j < argc; j++) { + if (SkStrStartsWith(argv[j], '-')) { + break; + } + help = false; + } + break; + } + } + if (!help) { + SkCommandLineFlags::Parse(argc, argv); + } else { + SkCommandLineFlags::PrintUsage(); + const char* commands[] = { "", "-h", "bmh", "-h", "examples", "-h", "include", "-h", "fiddle", + "-h", "ref", "-h", "tokens", + "-h", "crosscheck", "-h", "populate", "-h", "spellcheck" }; + SkCommandLineFlags::Parse(SK_ARRAY_COUNT(commands), (char**) commands); + return 0; + } + if (FLAGS_bmh.isEmpty() && FLAGS_include.isEmpty()) { + SkDebugf("requires -b or -i\n"); + SkCommandLineFlags::PrintUsage(); + return 1; + } + if (FLAGS_bmh.isEmpty() && !FLAGS_examples.isEmpty()) { + SkDebugf("-e requires -b\n"); + SkCommandLineFlags::PrintUsage(); + return 1; + } + if (FLAGS_hack) { + if (FLAGS_bmh.isEmpty()) { + SkDebugf("-k or --hack requires -b\n"); + SkCommandLineFlags::PrintUsage(); + return 1; + } + HackParser hacker; + if (!hacker.parseFile(FLAGS_bmh[0], ".bmh")) { + SkDebugf("hack failed\n"); + return -1; + } + SkDebugf("hack success\n"); + return 0; + } + if ((FLAGS_include.isEmpty() || FLAGS_bmh.isEmpty()) && FLAGS_populate) { + SkDebugf("-r requires -b -i\n"); + SkCommandLineFlags::PrintUsage(); + return 1; + } + if (FLAGS_bmh.isEmpty() && !FLAGS_ref.isEmpty()) { + SkDebugf("-r requires -b\n"); + SkCommandLineFlags::PrintUsage(); + return 1; + } + if (FLAGS_bmh.isEmpty() && FLAGS_spellcheck) { + SkDebugf("-s requires -b\n"); + SkCommandLineFlags::PrintUsage(); + return 1; + } + if (FLAGS_include.isEmpty() && FLAGS_tokens) { + SkDebugf("-t requires -i\n"); + SkCommandLineFlags::PrintUsage(); + return 1; + } + if ((FLAGS_include.isEmpty() || FLAGS_bmh.isEmpty()) && FLAGS_crosscheck) { + SkDebugf("-x requires -b -i\n"); + SkCommandLineFlags::PrintUsage(); + return 1; + } + if (!FLAGS_bmh.isEmpty()) { + if (!bmhParser.parseFile(FLAGS_bmh[0], ".bmh")) { + return -1; + } + } + bool done = false; + if (!FLAGS_include.isEmpty()) { + if (FLAGS_tokens || FLAGS_crosscheck) { + IncludeParser includeParser; + includeParser.validate(); + if (!includeParser.parseFile(FLAGS_include[0], ".h")) { + return -1; + } + if (FLAGS_tokens) { + includeParser.dumpTokens(); + done = true; + } else if (FLAGS_crosscheck) { + if (!includeParser.crossCheck(bmhParser)) { + return -1; + } + done = true; + } + } else if (FLAGS_populate) { + IncludeWriter includeWriter; + includeWriter.validate(); + if (!includeWriter.parseFile(FLAGS_include[0], ".h")) { + return -1; + } + if (!includeWriter.populate(bmhParser)) { + return -1; + } + done = true; + } + } + FiddleParser fparser(&bmhParser); + if (!done && !FLAGS_fiddle.isEmpty() && FLAGS_examples.isEmpty()) { + if (!fparser.parseFile(FLAGS_fiddle[0], ".txt")) { + return -1; + } + } + if (!done && !FLAGS_ref.isEmpty() && FLAGS_examples.isEmpty()) { + MdOut mdOut(bmhParser); + mdOut.buildReferences(FLAGS_bmh[0], FLAGS_ref[0]); + } + if (!done && FLAGS_spellcheck && FLAGS_examples.isEmpty()) { + bmhParser.spellCheck(FLAGS_bmh[0]); + done = true; + } + int examples = 0; + int methods = 0; + int topics = 0; + FILE* fiddleOut; + if (!done && !FLAGS_examples.isEmpty()) { + fiddleOut = fopen(FLAGS_examples[0], "wb"); + if (!fiddleOut) { + SkDebugf("could not open output file %s\n", FLAGS_examples[0]); + return -1; + } + fprintf(fiddleOut, "{\n"); + bool continuation = false; + for (const auto& topic : bmhParser.fTopicMap) { + if (topic.second->fParent) { + continue; + } + dump_examples(fiddleOut, *topic.second, &continuation); + } + fprintf(fiddleOut, "\n}\n"); + fclose(fiddleOut); + } + for (const auto& topic : bmhParser.fTopicMap) { + if (topic.second->fParent) { + continue; + } + examples += count_children(*topic.second, MarkType::kExample); + methods += count_children(*topic.second, MarkType::kMethod); + topics += count_children(*topic.second, MarkType::kSubtopic); + topics += count_children(*topic.second, MarkType::kTopic); + } + SkDebugf("topics=%d classes=%d methods=%d examples=%d\n", + bmhParser.fTopicMap.size(), bmhParser.fClassMap.size(), + methods, examples); + return 0; +} diff --git a/tools/bookmaker/bookmaker.h b/tools/bookmaker/bookmaker.h new file mode 100644 index 0000000000..8d9d14f6a5 --- /dev/null +++ b/tools/bookmaker/bookmaker.h @@ -0,0 +1,1844 @@ +/* + * Copyright 2017 Google Inc. + * + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#ifndef bookmaker_DEFINED +#define bookmaker_DEFINED + +#define STDOUT_TO_IDE_OUT 01 + +#include "SkData.h" + +#include <algorithm> +#include <cmath> +#include <cctype> +#include <forward_list> +#include <list> +#include <string> +#include <unordered_map> +#include <vector> + +// std::to_string isn't implemented on android +#include <sstream> + +template <typename T> +std::string to_string(T value) +{ + std::ostringstream os ; + os << value ; + return os.str() ; +} + +using std::forward_list; +using std::list; +using std::unordered_map; +using std::string; +using std::vector; + +enum class KeyWord { + kNone, + kBool, + kChar, + kClass, + kConst, + kConstExpr, + kDefine, + kDouble, + kElif, + kElse, + kEndif, + kEnum, + kFloat, + kFriend, + kIf, + kIfdef, + kIfndef, + kInclude, + kInline, + kInt, + kOperator, + kPrivate, + kProtected, + kPublic, + kSigned, + kSize_t, + kStatic, + kStruct, + kTemplate, + kTypedef, + kUint32_t, + kUnion, + kUnsigned, + kVoid, +}; + +enum class MarkType { + kNone, + kAnchor, + kAlias, + kBug, + kClass, + kCode, + kColumn, + kComment, + kConst, + kDefine, + kDefinedBy, + kDeprecated, + kDescription, + kDoxygen, + kEnum, + kEnumClass, + kError, + kExample, + kExperimental, + kExternal, + kFile, + kFormula, + kFunction, + kHeight, + kImage, + kLegend, + kLink, + kList, + kMarkChar, + kMember, + kMethod, + kNoExample, + kParam, + kPlatform, + kPrivate, + kReturn, + kRoot, + kRow, + kSeeAlso, + kStdOut, + kStruct, + kSubstitute, + kSubtopic, + kTable, + kTemplate, + kText, + kTime, + kToDo, + kTopic, + kTrack, + kTypedef, + kUnion, + kVolatile, + kWidth, +}; + +enum { + Last_MarkType = (int) MarkType::kWidth, +}; + +enum class Bracket { + kNone, + kParen, + kSquare, + kBrace, + kAngle, + kString, + kChar, + kSlashStar, + kSlashSlash, + kPound, + kColon, +}; + +enum class Punctuation { // catch-all for misc symbols tracked in C + kNone, + kAsterisk, // for pointer-to + kSemicolon, // e.g., to delinate xxx() const ; const int* yyy() + kLeftBrace, + kColon, // for foo() : bar(1), baz(2) {} +}; + +static inline bool has_nonwhitespace(const string& s) { + bool nonwhite = false; + for (const char& c : s) { + if (' ' < c) { + nonwhite = true; + break; + } + } + return nonwhite; +} + +static inline void trim_end(string &s) { + s.erase(std::find_if(s.rbegin(), s.rend(), + std::not1(std::ptr_fun<int, int>(std::isspace))).base(), s.end()); +} + +static inline void trim_end_spaces(string &s) { + while (s.length() > 0 && ' ' == s.back()) { + s.pop_back(); + } +} + +static inline void trim_start(string &s) { + s.erase(s.begin(), std::find_if(s.begin(), s.end(), + std::not1(std::ptr_fun<int, int>(std::isspace)))); +} + +static inline void trim_start_end(string& s) { + trim_start(s); + trim_end(s); +} + +class NonAssignable { +public: + NonAssignable(NonAssignable const&) = delete; + NonAssignable& operator=(NonAssignable const&) = delete; + NonAssignable() {} +}; + +class Definition; + +class TextParser : public NonAssignable { + TextParser() {} // only for ParserCommon to call + friend class ParserCommon; +public: + enum OneLine { + kNo, + kYes + }; + + class Save { + public: + Save(TextParser* parser) { + fParser = parser; + fLine = parser->fLine; + fChar = parser->fChar; + fLineCount = parser->fLineCount; + } + + void restore() const { + fParser->fLine = fLine; + fParser->fChar = fChar; + fParser->fLineCount = fLineCount; + } + + private: + TextParser* fParser; + const char* fLine; + const char* fChar; + int fLineCount; + }; + + TextParser(const string& fileName, const char* start, const char* end, int lineCount) + : fFileName(fileName) + , fStart(start) + , fLine(start) + , fChar(start) + , fEnd(end) + , fLineCount(lineCount) + { + } + + TextParser(const Definition* ); + + const char* anyOf(const char* str) const { + const char* ptr = fChar; + while (ptr < fEnd) { + if (strchr(str, ptr[0])) { + return ptr; + } + ++ptr; + } + return nullptr; + } + + const char* anyOf(const char* wordStart, const char* wordList[], size_t wordListCount) const { + const char** wordPtr = wordList; + const char** wordEnd = wordPtr + wordListCount; + const size_t matchLen = fChar - wordStart; + while (wordPtr < wordEnd) { + const char* word = *wordPtr++; + if (strlen(word) == matchLen && !strncmp(wordStart, word, matchLen)) { + return word; + } + } + return nullptr; + } + + char backup(const char* pattern) const { + size_t len = strlen(pattern); + const char* start = fChar - len; + if (start <= fStart) { + return '\0'; + } + if (strncmp(start, pattern, len)) { + return '\0'; + } + return start[-1]; + } + + bool contains(const char* match, const char* lineEnd, const char** loc) const { + *loc = this->strnstr(match, lineEnd); + return *loc; + } + + bool eof() const { return fChar >= fEnd; } + + const char* lineEnd() const { + const char* ptr = fChar; + do { + if (ptr >= fEnd) { + return ptr; + } + char test = *ptr++; + if (test == '\n' || test == '\0') { + break; + } + } while (true); + return ptr; + } + + ptrdiff_t lineLength() const { + return this->lineEnd() - fLine; + } + + bool match(TextParser* ); + + char next() { + SkASSERT(fChar < fEnd); + char result = *fChar++; + if ('\n' == result) { + ++fLineCount; + fLine = fChar; + } + return result; + } + + char peek() const { SkASSERT(fChar < fEnd); return *fChar; } + + void restorePlace(const TextParser& save) { + fChar = save.fChar; + fLine = save.fLine; + fLineCount = save.fLineCount; + } + + void savePlace(TextParser* save) { + save->fChar = fChar; + save->fLine = fLine; + save->fLineCount = fLineCount; + } + + void reportError(const char* errorStr) const; + void reportWarning(const char* errorStr) const; + + template <typename T> T reportError(const char* errorStr) const { + this->reportError(errorStr); + return T(); + } + + bool sentenceEnd(const char* check) const { + while (check > fStart) { + --check; + if (' ' < check[0] && '.' != check[0]) { + return false; + } + if ('.' == check[0]) { + return ' ' >= check[1]; + } + if ('\n' == check[0] && '\n' == check[1]) { + return true; + } + } + return true; + } + + bool skipToEndBracket(char endBracket, const char* end = nullptr) { + if (nullptr == end) { + end = fEnd; + } + while (fChar[0] != endBracket) { + if (fChar >= end) { + return false; + } + (void) this->next(); + } + return true; + } + + bool skipToEndBracket(const char* endBracket) { + size_t len = strlen(endBracket); + while (strncmp(fChar, endBracket, len)) { + if (fChar >= fEnd) { + return false; + } + (void) this->next(); + } + return true; + } + + bool skipLine() { + return skipToEndBracket('\n'); + } + + void skipTo(const char* skip) { + while (fChar < skip) { + this->next(); + } + } + + void skipToAlpha() { + while (fChar < fEnd && !isalpha(fChar[0])) { + fChar++; + } + } + + void skipToAlphaNum() { + while (fChar < fEnd && !isalnum(fChar[0])) { + fChar++; + } + } + + bool skipExact(const char* pattern) { + if (!this->startsWith(pattern)) { + return false; + } + this->skipName(pattern); + return true; + } + + // differs from skipToNonAlphaNum in that a.b isn't considered a full name, + // since a.b can't be found as a named definition + void skipFullName() { + while (fChar < fEnd && (isalnum(fChar[0]) + || '_' == fChar[0] || '-' == fChar[0] + || (':' == fChar[0] && fChar +1 < fEnd && ':' == fChar[1]))) { + if (':' == fChar[0] && fChar +1 < fEnd && ':' == fChar[1]) { + fChar++; + } + fChar++; + } + } + + bool skipToLineStart() { + if (!this->skipLine()) { + return false; + } + if (!this->eof()) { + return this->skipWhiteSpace(); + } + return true; + } + + void skipToNonAlphaNum() { + while (fChar < fEnd && (isalnum(fChar[0]) + || '_' == fChar[0] || '-' == fChar[0] + || (':' == fChar[0] && fChar +1 < fEnd && ':' == fChar[1]) + || ('.' == fChar[0] && fChar + 1 < fEnd && isalpha(fChar[1])))) { + if (':' == fChar[0] && fChar +1 < fEnd && ':' == fChar[1]) { + fChar++; + } + fChar++; + } + } + + void skipToSpace() { + while (fChar < fEnd && ' ' != fChar[0]) { + fChar++; + } + } + + bool skipName(const char* word) { + size_t len = strlen(word); + if (len < (size_t) (fEnd - fChar) && !strncmp(word, fChar, len)) { + fChar += len; + } + return this->eof() || ' ' >= fChar[0]; + } + + bool skipSpace() { + while (' ' == this->peek()) { + (void) this->next(); + if (fChar >= fEnd) { + return false; + } + } + return true; + } + + bool skipWord(const char* word) { + if (!this->skipWhiteSpace()) { + return false; + } + const char* save = fChar; + if (!this->skipName(word)) { + fChar = save; + return false; + } + if (!this->skipWhiteSpace()) { + return false; + } + return true; + } + + bool skipWhiteSpace() { + while (' ' >= this->peek()) { + (void) this->next(); + if (fChar >= fEnd) { + return false; + } + } + return true; + } + + bool startsWith(const char* str) const { + size_t len = strlen(str); + ptrdiff_t lineLen = this->lineLength(); + return len <= (size_t) lineLen && 0 == strncmp(str, fChar, len); + } + + const char* strnchr(char ch, const char* end) const { + const char* ptr = fChar; + while (ptr < end) { + if (ptr[0] == ch) { + return ptr; + } + ++ptr; + } + return nullptr; + } + + const char* strnstr(const char *match, const char* end) const { + size_t matchLen = strlen(match); + SkASSERT(matchLen > 0); + ptrdiff_t len = end - fChar; + SkASSERT(len >= 0); + if ((size_t) len < matchLen ) { + return nullptr; + } + size_t count = len - matchLen; + for (size_t index = 0; index <= count; index++) { + if (0 == strncmp(&fChar[index], match, matchLen)) { + return &fChar[index]; + } + } + return nullptr; + } + + const char* trimmedBracketEnd(const char bracket, OneLine oneLine) const { + int max = (int) (OneLine::kYes == oneLine ? this->lineLength() : fEnd - fChar); + int index = 0; + while (index < max && bracket != fChar[index]) { + ++index; + } + SkASSERT(index < max); + while (index > 0 && ' ' >= fChar[index - 1]) { + --index; + } + return fChar + index; + } + + const char* trimmedLineEnd() const { + const char* result = this->lineEnd(); + while (result > fChar && ' ' >= result[-1]) { + --result; + } + return result; + } + + void trimEnd() { + while (fEnd > fStart && ' ' >= fEnd[-1]) { + --fEnd; + } + } + + const char* wordEnd() const { + const char* end = fChar; + while (isalnum(end[0]) || '_' == end[0] || '-' == end[0]) { + ++end; + } + return end; + } + + string fFileName; + const char* fStart; + const char* fLine; + const char* fChar; + const char* fEnd; + size_t fLineCount; +}; + +class EscapeParser : public TextParser { +public: + EscapeParser(const char* start, const char* end) : + TextParser("", start, end, 0) { + const char* reader = fStart; + fStorage = new char[end - start]; + char* writer = fStorage; + while (reader < fEnd) { + char ch = *reader++; + if (ch != '\\') { + *writer++ = ch; + } else { + char ctrl = *reader++; + if (ctrl == 'u') { + unsigned unicode = 0; + for (int i = 0; i < 4; ++i) { + unicode <<= 4; + SkASSERT((reader[0] >= '0' && reader[0] <= '9') || + (reader[0] >= 'A' && reader[0] <= 'F')); + int nibble = *reader++ - '0'; + if (nibble > 9) { + nibble = 'A'- '9' + 1; + } + unicode |= nibble; + } + SkASSERT(unicode < 256); + *writer++ = (unsigned char) unicode; + } else { + SkASSERT(ctrl == 'n'); + *writer++ = '\n'; + } + } + } + fStart = fLine = fChar = fStorage; + fEnd = writer; + } + + virtual ~EscapeParser() { + delete fStorage; + } +private: + char* fStorage; +}; + +class RootDefinition; + +class Definition : public NonAssignable { +public: + enum Type { + kNone, + kWord, + kMark, + kKeyWord, + kBracket, + kPunctuation, + kFileType, + }; + + enum class TrimExtract { + kNo, + kYes + }; + + enum class MethodType { + kNone, + kConstructor, + kDestructor, + kOperator, + }; + + Definition() {} + + Definition(const char* start, const char* end, int line, Definition* parent) + : fStart(start) + , fContentStart(start) + , fContentEnd(end) + , fParent(parent) + , fLineCount(line) + , fType(Type::kWord) { + if (parent) { + SkASSERT(parent->fFileName.length() > 0); + fFileName = parent->fFileName; + } + this->setParentIndex(); + } + + Definition(MarkType markType, const char* start, int line, Definition* parent) + : Definition(markType, start, nullptr, line, parent) { + } + + Definition(MarkType markType, const char* start, const char* end, int line, Definition* parent) + : Definition(start, end, line, parent) { + fMarkType = markType; + fType = Type::kMark; + } + + Definition(Bracket bracket, const char* start, int lineCount, Definition* parent) + : Definition(start, nullptr, lineCount, parent) { + fBracket = bracket; + fType = Type::kBracket; + } + + Definition(KeyWord keyWord, const char* start, const char* end, int lineCount, + Definition* parent) + : Definition(start, end, lineCount, parent) { + fKeyWord = keyWord; + fType = Type::kKeyWord; + } + + Definition(Punctuation punctuation, const char* start, int lineCount, Definition* parent) + : Definition(start, nullptr, lineCount, parent) { + fPunctuation = punctuation; + fType = Type::kPunctuation; + } + + virtual ~Definition() {} + + virtual RootDefinition* asRoot() { SkASSERT(0); return nullptr; } + virtual const RootDefinition* asRoot() const { SkASSERT(0); return nullptr; } + + bool boilerplateIfDef(Definition* parent) { + const Definition& label = fTokens.front(); + if (Type::kWord != label.fType) { + return false; + } + fName = string(label.fContentStart, label.fContentEnd - label.fContentStart); + return true; + } + + // todo: this is matching #ifndef SkXXX_DEFINED for no particular reason + // it doesn't do anything useful with arbitrary input, e.g. #ifdef SK_SUPPORT_LEGACY_CANVAS_HELPERS +// also doesn't know what to do with SK_REQUIRE_LOCAL_VAR() + bool boilerplateDef(Definition* parent) { + if (!this->boilerplateIfDef(parent)) { + return false; + } + const char* s = fName.c_str(); + const char* e = strchr(s, '_'); + return true; // fixme: if this is trying to do something useful with define, do it here + if (!e) { + return false; + } + string prefix(s, e - s); + const char* inName = strstr(parent->fName.c_str(), prefix.c_str()); + if (!inName) { + return false; + } + if ('/' != inName[-1] && '\\' != inName[-1]) { + return false; + } + if (strcmp(inName + prefix.size(), ".h")) { + return false; + } + return true; + } + + bool boilerplateEndIf() { + return true; + } + + bool checkMethod() const; + + void setCanonicalFiddle(); + bool crossCheck(const char* tokenName, const Definition& includeToken) const; + bool crossCheck(const Definition& includeToken) const; + bool crossCheckInside(const char* start, const char* end, const Definition& includeToken) const; + bool exampleToScript(string* result) const; + + string extractText(TrimExtract trimExtract) const { + string result; + TextParser parser(fFileName, fContentStart, fContentEnd, fLineCount); + int childIndex = 0; + char mc = '#'; + while (parser.fChar < parser.fEnd) { + if (TrimExtract::kYes == trimExtract && !parser.skipWhiteSpace()) { + break; + } + if (parser.next() == mc) { + if (parser.next() == mc) { + if (parser.next() == mc) { + mc = parser.next(); + } + } else { + // fixme : more work to do if # style comment is in text + // if in method definition, could be alternate method name + --parser.fChar; + if (' ' < parser.fChar[0]) { + if (islower(parser.fChar[0])) { + result += '\n'; + parser.skipLine(); + } else { + SkASSERT(isupper(parser.fChar[0])); + parser.skipTo(fChildren[childIndex]->fTerminator); + if (mc == parser.fChar[0] && mc == parser.fChar[1]) { + parser.next(); + parser.next(); + } + childIndex++; + } + } else { + parser.skipLine(); + } + continue; + } + } else { + --parser.fChar; + } + const char* end = parser.fEnd; + const char* mark = parser.strnchr(mc, end); + if (mark) { + end = mark; + } + string fragment(parser.fChar, end - parser.fChar); + trim_end(fragment); + if (TrimExtract::kYes == trimExtract) { + trim_start(fragment); + if (result.length()) { + result += '\n'; + result += '\n'; + } + } + if (TrimExtract::kYes == trimExtract || has_nonwhitespace(fragment)) { + result += fragment; + } + parser.skipTo(end); + } + return result; + } + + string fiddleName() const; + string formatFunction() const; + const Definition* hasChild(MarkType markType) const; + const Definition* hasParam(const string& ref) const; + bool isClone() const { return fClone; } + + Definition* iRootParent() { + Definition* test = fParent; + while (test) { + if (Type::kKeyWord == test->fType && KeyWord::kClass == test->fKeyWord) { + return test; + } + test = test->fParent; + } + return nullptr; + } + + virtual bool isRoot() const { return false; } + + int length() const { + return (int) (fContentEnd - fContentStart); + } + + bool methodHasReturn(const string& name, TextParser* methodParser) const; + string methodName() const; + bool nextMethodParam(TextParser* methodParser, const char** nextEndPtr, + string* paramName) const; + bool paramsMatch(const string& fullRef, const string& name) const; + + string printableName() const { + string result(fName); + std::replace(result.begin(), result.end(), '_', ' '); + return result; + } + + virtual RootDefinition* rootParent() { SkASSERT(0); return nullptr; } + + void setParentIndex() { + fParentIndex = fParent ? (int) fParent->fTokens.size() : -1; + } + + string fText; // if text is constructed instead of in a file, it's put here + const char* fStart = nullptr; // .. in original text file, or the start of fText + const char* fContentStart; // start past optional markup name + string fName; + string fFiddle; // if its a constructor or operator, fiddle name goes here + const char* fContentEnd = nullptr; // the end of the contained text + const char* fTerminator = nullptr; // the end of the markup, normally ##\n or \n + Definition* fParent = nullptr; + list<Definition> fTokens; + vector<Definition*> fChildren; + string fHash; // generated by fiddle + string fFileName; + size_t fLineCount = 0; + int fParentIndex = 0; + MarkType fMarkType = MarkType::kNone; + KeyWord fKeyWord = KeyWord::kNone; + Bracket fBracket = Bracket::kNone; + Punctuation fPunctuation = Punctuation::kNone; + MethodType fMethodType = MethodType::kNone; + Type fType = Type::kNone; + bool fClone = false; + bool fCloned = false; + bool fPrivate = false; + bool fShort = false; + bool fMemberStart = false; + mutable bool fVisited = false; +}; + +class RootDefinition : public Definition { +public: + RootDefinition() { + } + + RootDefinition(MarkType markType, const char* start, int line, Definition* parent) + : Definition(markType, start, line, parent) { + } + + RootDefinition(MarkType markType, const char* start, const char* end, int line, + Definition* parent) : Definition(markType, start, end, line, parent) { + } + + ~RootDefinition() override { + for (auto& iter : fBranches) { + delete iter.second; + } + } + + RootDefinition* asRoot() override { return this; } + const RootDefinition* asRoot() const override { return this; } + void clearVisited(); + bool dumpUnVisited(); + const Definition* find(const string& ref) const; + bool isRoot() const override { return true; } + RootDefinition* rootParent() override { return fRootParent; } + void setRootParent(RootDefinition* rootParent) { fRootParent = rootParent; } + + unordered_map<string, RootDefinition*> fBranches; + unordered_map<string, Definition> fLeaves; +private: + RootDefinition* fRootParent = nullptr; +}; + +struct IClassDefinition : public Definition { + unordered_map<string, Definition*> fEnums; + unordered_map<string, Definition*> fMembers; + unordered_map<string, Definition*> fMethods; + unordered_map<string, Definition*> fStructs; +}; + +struct Reference { + Reference() + : fLocation(nullptr) + , fDefinition(nullptr) { + } + + const char* fLocation; // .. in original text file + const Definition* fDefinition; +}; + +struct TypeNames { + const char* fName; + MarkType fMarkType; +}; + +class ParserCommon : public TextParser { +public: + + ParserCommon() : TextParser() + , fParent(nullptr) + { + } + + virtual ~ParserCommon() { + } + + void addDefinition(Definition* def) { + fParent->fChildren.push_back(def); + fParent = def; + } + + void indentToColumn(int column) { + SkASSERT(column >= fColumn); +#if STDOUT_TO_IDE_OUT + SkDebugf("%*s", column - fColumn, ""); +#endif + fprintf(fOut, "%*s", column - fColumn, ""); + fColumn = column; + fSpaces += column - fColumn; + } + + bool leadingPunctuation(const char* str, size_t len) const { + if (!fPendingSpace) { + return false; + } + if (len < 2) { + return false; + } + if ('.' != str[0] && ',' != str[0] && ';' != str[0] && ':' != str[0]) { + return false; + } + return ' ' >= str[1]; + } + + void lf(int count) { + fPendingLF = SkTMax(fPendingLF, count); + this->nl(); + } + + void lfAlways(int count) { + this->lf(count); + this->writePending(); + } + + void lfcr() { + this->lf(1); + } + + void nl() { + fLinefeeds = 0; + fSpaces = 0; + fColumn = 0; + fPendingSpace = false; + } + + bool parseFile(const char* file, const char* suffix); + virtual bool parseFromFile(const char* path) = 0; + bool parseSetup(const char* path); + + void popObject() { + fParent->fContentEnd = fParent->fTerminator = fChar; + fParent = fParent->fParent; + } + + virtual void reset() = 0; + + void resetCommon() { + fLine = fChar = fStart; + fLineCount = 0; + fParent = nullptr; + fIndent = 0; + fOut = nullptr; + fMaxLF = 2; + fPendingLF = 0; + fPendingSpace = false; + nl(); + } + + void setAsParent(Definition* definition) { + if (fParent) { + fParent->fChildren.push_back(definition); + definition->fParent = fParent; + } + fParent = definition; + } + + void singleLF() { + fMaxLF = 1; + } + + bool writeBlockTrim(int size, const char* data) { + while (size && ' ' >= data[0]) { + ++data; + --size; + } + while (size && ' ' >= data[size - 1]) { + --size; + } + if (size <= 0) { + return false; + } + SkASSERT(size < 8000); + if (size > 3 && !strncmp("#end", data, 4)) { + fMaxLF = 1; + } + if (this->leadingPunctuation(data, (size_t) size)) { + fPendingSpace = false; + } + writePending(); +#if STDOUT_TO_IDE_OUT + string check(data, size); + SkDebugf("%s", check.c_str()); +#endif + fprintf(fOut, "%.*s", size, data); + int added = 0; + while (size > 0 && '\n' != data[--size]) { + ++added; + } + fColumn = size ? added : fColumn + added; + fSpaces = 0; + fLinefeeds = 0; + fMaxLF = added > 2 && !strncmp("#if", &data[size + (size > 0)], 3) ? 1 : 2; + return true; + } + + void writeBlock(int size, const char* data) { + SkAssertResult(writeBlockTrim(size, data)); + } + void writeCommentHeader() { + this->lf(2); + this->writeString("/**"); + this->writeSpace(); + } + + void writeCommentTrailer() { + this->writeString("*/"); + this->lfcr(); + } + + // write a pending space, so that two consecutive calls + // don't double write, and trailing spaces on lines aren't written + void writeSpace() { + SkASSERT(!fPendingLF); + SkASSERT(!fLinefeeds); + SkASSERT(fColumn > 0); + SkASSERT(!fSpaces); + fPendingSpace = true; + } + + void writeString(const char* str) { + SkASSERT(strlen(str) > 0); + SkASSERT(' ' < str[0]); + SkASSERT(' ' < str[strlen(str) - 1]); + if (this->leadingPunctuation(str, strlen(str))) { + fPendingSpace = false; + } + writePending(); +#if STDOUT_TO_IDE_OUT + SkDebugf("%s", str); +#endif + SkASSERT(!strchr(str, '\n')); + fprintf(fOut, "%s", str); + fColumn += strlen(str); + fSpaces = 0; + fLinefeeds = 0; + fMaxLF = 2; + } + + void writePending() { + fPendingLF = SkTMin(fPendingLF, fMaxLF); + bool wroteLF = false; + while (fLinefeeds < fPendingLF) { +#if STDOUT_TO_IDE_OUT + SkDebugf("\n"); +#endif + fprintf(fOut, "\n"); + ++fLinefeeds; + wroteLF = true; + } + fPendingLF = 0; + if (wroteLF) { + SkASSERT(0 == fColumn); + SkASSERT(fIndent >= fSpaces); + #if STDOUT_TO_IDE_OUT + SkDebugf("%*s", fIndent - fSpaces, ""); + #endif + fprintf(fOut, "%*s", fIndent - fSpaces, ""); + fColumn = fIndent; + fSpaces = fIndent; + } + if (fPendingSpace) { + #if STDOUT_TO_IDE_OUT + SkDebugf(" "); + #endif + fprintf(fOut, " "); + ++fColumn; + fPendingSpace = false; + } + } + + unordered_map<string, sk_sp<SkData>> fRawData; + unordered_map<string, vector<char>> fLFOnly; + Definition* fParent; + FILE* fOut; + int fLinefeeds; // number of linefeeds last written, zeroed on non-space + int fMaxLF; // number of linefeeds allowed + int fPendingLF; // number of linefeeds to write (can be suppressed) + int fSpaces; // number of spaces (indent) last written, zeroed on non-space + int fColumn; // current column; number of chars past last linefeed + int fIndent; // desired indention + bool fPendingSpace; // a space should preceed the next string or block +private: + typedef TextParser INHERITED; +}; + + + +class BmhParser : public ParserCommon { +public: + enum class MarkLookup { + kRequire, + kAllowUnknown, + }; + + enum class Resolvable { + kNo, // neither resolved nor output + kYes, // resolved, output + kOut, // not resolved, but output + }; + + enum class Exemplary { + kNo, + kYes, + kOptional, + }; + +#define M(mt) (1LL << (int) MarkType::k##mt) +#define M_D M(Description) +#define M_CS M(Class) | M(Struct) +#define M_ST M(Subtopic) | M(Topic) +#define M_CSST M_CS | M_ST +#ifdef M_E +#undef M_E +#endif +#define M_E M(Enum) | M(EnumClass) + +#define R_Y Resolvable::kYes +#define R_N Resolvable::kNo +#define R_O Resolvable::kOut + +#define E_Y Exemplary::kYes +#define E_N Exemplary::kNo +#define E_O Exemplary::kOptional + + BmhParser() : ParserCommon() + , fMaps { +// names without formal definitions (e.g. Column) aren't included +// fill in other names once they're actually used + { "", nullptr, MarkType::kNone, R_Y, E_N, 0 } +, { "A", nullptr, MarkType::kAnchor, R_Y, E_N, 0 } +, { "Alias", nullptr, MarkType::kAlias, R_N, E_N, 0 } +, { "Bug", nullptr, MarkType::kBug, R_N, E_N, 0 } +, { "Class", &fClassMap, MarkType::kClass, R_Y, E_O, M_CSST | M(Root) } +, { "Code", nullptr, MarkType::kCode, R_Y, E_N, M_CSST | M_E } +, { "", nullptr, MarkType::kColumn, R_Y, E_N, M(Row) } +, { "", nullptr, MarkType::kComment, R_N, E_N, 0 } +, { "Const", &fConstMap, MarkType::kConst, R_Y, E_N, M_E | M_ST } +, { "Define", nullptr, MarkType::kDefine, R_O, E_N, M_ST } +, { "DefinedBy", nullptr, MarkType::kDefinedBy, R_N, E_N, M(Method) } +, { "Deprecated", nullptr, MarkType::kDeprecated, R_Y, E_N, 0 } +, { "Description", nullptr, MarkType::kDescription, R_Y, E_N, M(Example) } +, { "Doxygen", nullptr, MarkType::kDoxygen, R_Y, E_N, 0 } +, { "Enum", &fEnumMap, MarkType::kEnum, R_Y, E_O, M_CSST | M(Root) } +, { "EnumClass", &fClassMap, MarkType::kEnumClass, R_Y, E_O, M_CSST | M(Root) } +, { "Error", nullptr, MarkType::kError, R_N, E_N, M(Example) } +, { "Example", nullptr, MarkType::kExample, R_O, E_N, M_CSST | M_E | M(Method) } +, { "Experimental", nullptr, MarkType::kExperimental, R_Y, E_N, 0 } +, { "External", nullptr, MarkType::kExternal, R_Y, E_N, M(Root) } +, { "File", nullptr, MarkType::kFile, R_N, E_N, M(Track) } +, { "Formula", nullptr, MarkType::kFormula, R_O, E_N, M_ST | M(Method) | M_D } +, { "Function", nullptr, MarkType::kFunction, R_O, E_N, M(Example) } +, { "Height", nullptr, MarkType::kHeight, R_N, E_N, M(Example) } +, { "Image", nullptr, MarkType::kImage, R_N, E_N, M(Example) } +, { "Legend", nullptr, MarkType::kLegend, R_Y, E_N, M(Table) } +, { "", nullptr, MarkType::kLink, R_Y, E_N, M(Anchor) } +, { "List", nullptr, MarkType::kList, R_Y, E_N, M(Method) | M_CSST | M_E | M_D } +, { "", nullptr, MarkType::kMarkChar, R_N, E_N, 0 } +, { "Member", nullptr, MarkType::kMember, R_Y, E_N, M(Class) | M(Struct) } +, { "Method", &fMethodMap, MarkType::kMethod, R_Y, E_Y, M_CSST } +, { "NoExample", nullptr, MarkType::kNoExample, R_Y, E_N, 0 } +, { "Param", nullptr, MarkType::kParam, R_Y, E_N, M(Method) } +, { "Platform", nullptr, MarkType::kPlatform, R_Y, E_N, M(Example) } +, { "Private", nullptr, MarkType::kPrivate, R_N, E_N, 0 } +, { "Return", nullptr, MarkType::kReturn, R_Y, E_N, M(Method) } +, { "", nullptr, MarkType::kRoot, R_Y, E_N, 0 } +, { "", nullptr, MarkType::kRow, R_Y, E_N, M(Table) | M(List) } +, { "SeeAlso", nullptr, MarkType::kSeeAlso, R_Y, E_N, M_CSST | M_E | M(Method) } +, { "StdOut", nullptr, MarkType::kStdOut, R_N, E_N, M(Example) } +, { "Struct", &fClassMap, MarkType::kStruct, R_Y, E_O, M(Class) | M(Root) | M_ST } +, { "Substitute", nullptr, MarkType::kSubstitute, R_N, E_N, M_ST } +, { "Subtopic", nullptr, MarkType::kSubtopic, R_Y, E_Y, M_CSST } +, { "Table", nullptr, MarkType::kTable, R_Y, E_N, M(Method) | M_CSST | M_E } +, { "Template", nullptr, MarkType::kTemplate, R_Y, E_N, 0 } +, { "", nullptr, MarkType::kText, R_Y, E_N, 0 } +, { "Time", nullptr, MarkType::kTime, R_Y, E_N, M(Track) } +, { "ToDo", nullptr, MarkType::kToDo, R_N, E_N, 0 } +, { "Topic", nullptr, MarkType::kTopic, R_Y, E_Y, M_CS | M(Root) | M(Topic) } +, { "Track", nullptr, MarkType::kTrack, R_Y, E_N, M_E | M_ST } +, { "Typedef", &fTypedefMap, MarkType::kTypedef, R_Y, E_N, M(Subtopic) | M(Topic) } +, { "", nullptr, MarkType::kUnion, R_Y, E_N, 0 } +, { "Volatile", nullptr, MarkType::kVolatile, R_N, E_N, M(StdOut) } +, { "Width", nullptr, MarkType::kWidth, R_N, E_N, M(Example) } } + { + this->reset(); + } + +#undef R_O +#undef R_N +#undef R_Y + +#undef M_E +#undef M_CSST +#undef M_ST +#undef M_CS +#undef M_D +#undef M + + ~BmhParser() override {} + + bool addDefinition(const char* defStart, bool hasEnd, MarkType markType, + const vector<string>& typeNameBuilder); + bool childOf(MarkType markType) const; + string className(MarkType markType); + bool collectExternals(); + int endHashCount() const; + + RootDefinition* findBmhObject(MarkType markType, const string& typeName) { + auto map = fMaps[(int) markType].fBmh; + if (!map) { + return nullptr; + } + return &(*map)[typeName]; + } + + bool findDefinitions(); + MarkType getMarkType(MarkLookup lookup) const; + bool hasEndToken() const; + string memberName(); + string methodName(); + const Definition* parentSpace() const; + + bool parseFromFile(const char* path) override { + if (!INHERITED::parseSetup(path)) { + return false; + } + fCheckMethods = !strstr(path, "undocumented.bmh"); + return findDefinitions(); + } + + bool popParentStack(Definition* definition); + + void reset() override { + INHERITED::resetCommon(); + fRoot = nullptr; + fMC = '#'; + fInChar = false; + fInCharCommentString = false; + fInComment = false; + fInEnum = false; + fInString = false; + fCheckMethods = false; + } + + bool skipNoName(); + bool skipToDefinitionEnd(MarkType markType); + void spellCheck(const char* match) const; + vector<string> topicName(); + vector<string> typeName(MarkType markType, bool* expectEnd); + string uniqueName(const string& base, MarkType markType); + string uniqueRootName(const string& base, MarkType markType); + void validate() const; + string word(const string& prefix, const string& delimiter); + +public: + struct DefinitionMap { + const char* fName; + unordered_map<string, RootDefinition>* fBmh; + MarkType fMarkType; + Resolvable fResolve; + Exemplary fExemplary; // worthy of an example + uint64_t fParentMask; + }; + + DefinitionMap fMaps[Last_MarkType + 1]; + forward_list<RootDefinition> fTopics; + forward_list<Definition> fMarkup; + forward_list<RootDefinition> fExternals; + vector<string> fInputFiles; + unordered_map<string, RootDefinition> fClassMap; + unordered_map<string, RootDefinition> fConstMap; + unordered_map<string, RootDefinition> fEnumMap; + unordered_map<string, RootDefinition> fMethodMap; + unordered_map<string, RootDefinition> fTypedefMap; + unordered_map<string, Definition*> fTopicMap; + unordered_map<string, Definition*> fAliasMap; + RootDefinition* fRoot; + mutable char fMC; // markup character + bool fAnonymous; + bool fCloned; + bool fInChar; + bool fInCharCommentString; + bool fInEnum; + bool fInComment; + bool fInString; + bool fCheckMethods; + +private: + typedef ParserCommon INHERITED; +}; + +class IncludeParser : public ParserCommon { +public: + enum class IsStruct { + kNo, + kYes, + }; + + IncludeParser() : ParserCommon() + , fMaps { + { nullptr, MarkType::kNone } + , { nullptr, MarkType::kAnchor } + , { nullptr, MarkType::kAlias } + , { nullptr, MarkType::kBug } + , { nullptr, MarkType::kClass } + , { nullptr, MarkType::kCode } + , { nullptr, MarkType::kColumn } + , { nullptr, MarkType::kComment } + , { nullptr, MarkType::kConst } + , { &fIDefineMap, MarkType::kDefine } + , { nullptr, MarkType::kDefinedBy } + , { nullptr, MarkType::kDeprecated } + , { nullptr, MarkType::kDescription } + , { nullptr, MarkType::kDoxygen } + , { &fIEnumMap, MarkType::kEnum } + , { &fIEnumMap, MarkType::kEnumClass } + , { nullptr, MarkType::kError } + , { nullptr, MarkType::kExample } + , { nullptr, MarkType::kExperimental } + , { nullptr, MarkType::kExternal } + , { nullptr, MarkType::kFile } + , { nullptr, MarkType::kFormula } + , { nullptr, MarkType::kFunction } + , { nullptr, MarkType::kHeight } + , { nullptr, MarkType::kImage } + , { nullptr, MarkType::kLegend } + , { nullptr, MarkType::kLink } + , { nullptr, MarkType::kList } + , { nullptr, MarkType::kMarkChar } + , { nullptr, MarkType::kMember } + , { nullptr, MarkType::kMethod } + , { nullptr, MarkType::kNoExample } + , { nullptr, MarkType::kParam } + , { nullptr, MarkType::kPlatform } + , { nullptr, MarkType::kPrivate } + , { nullptr, MarkType::kReturn } + , { nullptr, MarkType::kRoot } + , { nullptr, MarkType::kRow } + , { nullptr, MarkType::kSeeAlso } + , { nullptr, MarkType::kStdOut } + , { &fIStructMap, MarkType::kStruct } + , { nullptr, MarkType::kSubstitute } + , { nullptr, MarkType::kSubtopic } + , { nullptr, MarkType::kTable } + , { &fITemplateMap, MarkType::kTemplate } + , { nullptr, MarkType::kText } + , { nullptr, MarkType::kTime } + , { nullptr, MarkType::kToDo } + , { nullptr, MarkType::kTopic } + , { nullptr, MarkType::kTrack } + , { &fITypedefMap, MarkType::kTypedef } + , { &fIUnionMap, MarkType::kUnion } + , { nullptr, MarkType::kVolatile } + , { nullptr, MarkType::kWidth } } + { + this->reset(); + } + + ~IncludeParser() override {} + + void addKeyword(KeyWord keyWord); + + void addPunctuation(Punctuation punctuation) { + fParent->fTokens.emplace_back(punctuation, fChar, fLineCount, fParent); + } + + void addWord() { + fParent->fTokens.emplace_back(fIncludeWord, fChar, fLineCount, fParent); + fIncludeWord = nullptr; + } + + void checkForMissingParams(const vector<string>& methodParams, + const vector<string>& foundParams); + bool checkForWord(); + string className() const; + bool crossCheck(BmhParser& ); + IClassDefinition* defineClass(const Definition& includeDef, const string& className); + void dumpClassTokens(IClassDefinition& classDef); + void dumpComment(Definition* token); + void dumpTokens(); + bool findComments(const Definition& includeDef, Definition* markupDef); + + Definition* findIncludeObject(const Definition& includeDef, MarkType markType, + const string& typeName) { + typedef Definition* DefinitionPtr; + unordered_map<string, Definition>* map = fMaps[(int) markType].fInclude; + if (!map) { + return reportError<DefinitionPtr>("invalid mark type"); + } + string name = this->uniqueName(*map, typeName); + Definition& markupDef = (*map)[name]; + if (markupDef.fStart) { + return reportError<DefinitionPtr>("definition already defined"); + } + markupDef.fFileName = fFileName; + markupDef.fStart = includeDef.fStart; + markupDef.fContentStart = includeDef.fStart; + markupDef.fName = name; + markupDef.fContentEnd = includeDef.fContentEnd; + markupDef.fTerminator = includeDef.fTerminator; + markupDef.fParent = fParent; + markupDef.fLineCount = includeDef.fLineCount; + markupDef.fMarkType = markType; + markupDef.fKeyWord = includeDef.fKeyWord; + markupDef.fType = Definition::Type::kMark; + return &markupDef; + } + + static KeyWord FindKey(const char* start, const char* end); + void keywordEnd(); + void keywordStart(const char* keyword); + bool parseChar(); + bool parseComment(const string& filename, const char* start, const char* end, int lineCount, + Definition* markupDef); + bool parseClass(Definition* def, IsStruct); + bool parseDefine(); + bool parseEnum(Definition* child, Definition* markupDef); + + bool parseFromFile(const char* path) override { + if (!INHERITED::parseSetup(path)) { + return false; + } + string name(path); + return parseInclude(name); + } + + bool parseInclude(const string& name); + bool parseMember(Definition* child, Definition* markupDef); + bool parseMethod(Definition* child, Definition* markupDef); + bool parseObject(Definition* child, Definition* markupDef); + bool parseObjects(Definition* parent, Definition* markupDef); + bool parseTemplate(); + bool parseTypedef(); + bool parseUnion(); + + void popBracket() { + SkASSERT(Definition::Type::kBracket == fParent->fType); + this->popObject(); + Bracket bracket = this->topBracket(); + this->setBracketShortCuts(bracket); + } + + void pushBracket(Bracket bracket) { + this->setBracketShortCuts(bracket); + fParent->fTokens.emplace_back(bracket, fChar, fLineCount, fParent); + Definition* container = &fParent->fTokens.back(); + this->addDefinition(container); + } + + void reset() override { + INHERITED::resetCommon(); + fRootTopic = nullptr; + fInBrace = nullptr; + fIncludeWord = nullptr; + fPrev = '\0'; + fInChar = false; + fInCharCommentString = false; + fInComment = false; + fInEnum = false; + fInFunction = false; + fInString = false; + } + + void setBracketShortCuts(Bracket bracket) { + fInComment = Bracket::kSlashSlash == bracket || Bracket::kSlashStar == bracket; + fInString = Bracket::kString == bracket; + fInChar = Bracket::kChar == bracket; + fInCharCommentString = fInChar || fInComment || fInString; + } + + Bracket topBracket() const { + Definition* parent = fParent; + while (parent && Definition::Type::kBracket != parent->fType) { + parent = parent->fParent; + } + return parent ? parent->fBracket : Bracket::kNone; + } + + template <typename T> + string uniqueName(const unordered_map<string, T>& m, const string& typeName) { + string base(typeName.size() > 0 ? typeName : "_anonymous"); + string name(base); + int anonCount = 1; + do { + auto iter = m.find(name); + if (iter == m.end()) { + return name; + } + name = base + '_'; + name += to_string(++anonCount); + } while (true); + // should never get here + return string(); + } + + void validate() const; + +protected: + static void ValidateKeyWords(); + + struct DefinitionMap { + unordered_map<string, Definition>* fInclude; + MarkType fMarkType; + }; + + DefinitionMap fMaps[Last_MarkType + 1]; + unordered_map<string, Definition> fIncludeMap; + unordered_map<string, IClassDefinition> fIClassMap; + unordered_map<string, Definition> fIDefineMap; + unordered_map<string, Definition> fIEnumMap; + unordered_map<string, Definition> fIStructMap; + unordered_map<string, Definition> fITemplateMap; + unordered_map<string, Definition> fITypedefMap; + unordered_map<string, Definition> fIUnionMap; + Definition* fRootTopic; + Definition* fInBrace; + const char* fIncludeWord; + char fPrev; + bool fInChar; + bool fInCharCommentString; + bool fInComment; + bool fInEnum; + bool fInFunction; + bool fInString; + + typedef ParserCommon INHERITED; +}; + +class IncludeWriter : public IncludeParser { +public: + enum class Word { + kStart, + kCap, + kFirst, + kUnderline, + kMixed, + }; + + enum class PunctuationState { + kStart, + kDelimiter, + kPeriod, + }; + + enum class Wrote { + kNone, + kLF, + kChars, + }; + + IncludeWriter() : IncludeParser() {} + ~IncludeWriter() override {} + + bool contentFree(int size, const char* data) const { + while (size > 0 && data[0] <= ' ') { + --size; + ++data; + } + while (size > 0 && data[size - 1] <= ' ') { + --size; + } + return 0 == size; + } + + void enumHeaderOut(const RootDefinition* root, const Definition& child); + void enumMembersOut(const RootDefinition* root, const Definition& child); + void enumSizeItems(const Definition& child); + int lookupMethod(const PunctuationState punctuation, const Word word, + const int start, const int run, int lastWrite, const char last, + const char* data); + int lookupReference(const PunctuationState punctuation, const Word word, + const int start, const int run, int lastWrite, const char last, + const char* data); + void methodOut(const Definition* method); + bool populate(Definition* def, RootDefinition* root); + bool populate(BmhParser& bmhParser); + + void reset() override { + INHERITED::resetCommon(); + fBmhParser = nullptr; + fEnumDef = nullptr; + fStructDef = nullptr; + fAnonymousEnumCount = 1; + fInStruct = false; + } + + string resolveMethod(const char* start, const char* end, bool first); + string resolveRef(const char* start, const char* end, bool first); + Wrote rewriteBlock(int size, const char* data); + void structMemberOut(const Definition* memberStart, const Definition& child); + void structOut(const Definition* root, const Definition& child, + const char* commentStart, const char* commentEnd); + void structSizeMembers(Definition& child); + +private: + BmhParser* fBmhParser; + Definition* fDeferComment; + const Definition* fEnumDef; + const Definition* fStructDef; + const char* fContinuation; // used to construct paren-qualified method name + int fAnonymousEnumCount; + int fEnumItemValueTab; + int fEnumItemCommentTab; + int fStructMemberTab; + int fStructCommentTab; + bool fInStruct; + + typedef IncludeParser INHERITED; +}; + +class FiddleParser : public ParserCommon { +public: + FiddleParser(BmhParser* bmh) : ParserCommon() + , fBmhParser(bmh) { + this->reset(); + } + + Definition* findExample(const string& name) const; + + bool parseFromFile(const char* path) override { + if (!INHERITED::parseSetup(path)) { + return false; + } + return parseFiddles(); + } + + void reset() override { + INHERITED::resetCommon(); + } + +private: + bool parseFiddles(); + + BmhParser* fBmhParser; // must be writable; writes example hash + + typedef ParserCommon INHERITED; +}; + +class HackParser : public ParserCommon { +public: + HackParser() : ParserCommon() { + this->reset(); + } + + bool parseFromFile(const char* path) override { + if (!INHERITED::parseSetup(path)) { + return false; + } + return hackFiles(); + } + + void reset() override { + INHERITED::resetCommon(); + } + +private: + bool hackFiles(); + + typedef ParserCommon INHERITED; +}; + +class MdOut : public ParserCommon { +public: + MdOut(const BmhParser& bmh) : ParserCommon() + , fBmhParser(bmh) { + this->reset(); + } + + bool buildReferences(const char* path, const char* outDir); +private: + enum class TableState { + kNone, + kRow, + kColumn, + }; + + string addReferences(const char* start, const char* end, BmhParser::Resolvable ); + bool buildRefFromFile(const char* fileName, const char* outDir); + void childrenOut(const Definition* def, const char* contentStart); + const Definition* isDefined(const TextParser& parser, const string& ref, bool report) const; + string linkName(const Definition* ) const; + string linkRef(const string& leadingSpaces, const Definition*, const string& ref) const; + void markTypeOut(Definition* ); + void mdHeaderOut(int depth) { mdHeaderOutLF(depth, 2); } + void mdHeaderOutLF(int depth, int lf); + bool parseFromFile(const char* path) override { + return true; + } + + void reset() override { + INHERITED::resetCommon(); + fMethod = nullptr; + fRoot = nullptr; + fTableState = TableState::kNone; + fHasFiddle = false; + fInDescription = false; + fInList = false; + } + + BmhParser::Resolvable resolvable(MarkType markType) { + if ((MarkType::kExample == markType + || MarkType::kFunction == markType) && fHasFiddle) { + return BmhParser::Resolvable::kNo; + } + return fBmhParser.fMaps[(int) markType].fResolve; + } + + void resolveOut(const char* start, const char* end, BmhParser::Resolvable ); + + const BmhParser& fBmhParser; + Definition* fMethod; + RootDefinition* fRoot; + TableState fTableState; + bool fHasFiddle; + bool fInDescription; // FIXME: for now, ignore unfound camelCase in description since it may + // be defined in example which at present cannot be linked to + bool fInList; + typedef ParserCommon INHERITED; +}; + + +// some methods cannot be trivially parsed; look for class-name / ~ / operator +class MethodParser : public TextParser { +public: + MethodParser(const string& className, const string& fileName, + const char* start, const char* end, int lineCount) + : TextParser(fileName, start, end, lineCount) + , fClassName(className) { + } + + void skipToMethodStart() { + if (!fClassName.length()) { + this->skipToAlphaNum(); + return; + } + while (!this->eof() && !isalnum(this->peek()) && '~' != this->peek()) { + this->next(); + } + } + + void skipToMethodEnd() { + if (this->eof()) { + return; + } + if (fClassName.length()) { + if ('~' == this->peek()) { + this->next(); + if (!this->startsWith(fClassName.c_str())) { + --fChar; + return; + } + } + if (this->startsWith(fClassName.c_str()) || this->startsWith("operator")) { + const char* ptr = this->anyOf(" ("); + if (ptr && '(' == *ptr) { + this->skipToEndBracket(')'); + SkAssertResult(')' == this->next()); + return; + } + } + } + if (this->startsWith("Sk") && this->wordEndsWith(".h")) { // allow include refs + this->skipToNonAlphaNum(); + } else { + this->skipFullName(); + } + } + + bool wordEndsWith(const char* str) const { + const char* space = this->strnchr(' ', fEnd); + if (!space) { + return false; + } + size_t len = strlen(str); + if (space < fChar + len) { + return false; + } + return !strncmp(str, space - len, len); + } + +private: + string fClassName; + typedef TextParser INHERITED; +}; + +#endif diff --git a/tools/bookmaker/fiddleParser.cpp b/tools/bookmaker/fiddleParser.cpp new file mode 100644 index 0000000000..d5cfcf425c --- /dev/null +++ b/tools/bookmaker/fiddleParser.cpp @@ -0,0 +1,231 @@ +/* + * Copyright 2017 Google Inc. + * + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include "bookmaker.h" + +static Definition* find_fiddle(Definition* def, const string& name) { + if (MarkType::kExample == def->fMarkType && name == def->fFiddle) { + return def; + } + for (auto& child : def->fChildren) { + Definition* result = find_fiddle(child, name); + if (result) { + return result; + } + } + return nullptr; +} + +Definition* FiddleParser::findExample(const string& name) const { + for (const auto& topic : fBmhParser->fTopicMap) { + if (topic.second->fParent) { + continue; + } + Definition* def = find_fiddle(topic.second, name); + if (def) { + return def; + } + } + return nullptr; +} + +bool FiddleParser::parseFiddles() { + if (!this->skipExact("{\n")) { + return false; + } + while (!this->eof()) { + if (!this->skipExact(" \"")) { + return false; + } + const char* nameLoc = fChar; + if (!this->skipToEndBracket("\"")) { + return false; + } + string name(nameLoc, fChar - nameLoc); + if (!this->skipExact("\": {\n")) { + return false; + } + if (!this->skipExact(" \"compile_errors\": [")) { + return false; + } + if (']' != this->peek()) { + // report compiler errors + int brackets = 1; + const char* errorStart = fChar; + do { + if ('[' == this->peek()) { + ++brackets; + } else if (']' == this->peek()) { + --brackets; + } + } while (!this->eof() && this->next() && brackets > 0); + SkDebugf("fiddle compile error in %s: %.*s\n", name.c_str(), (int) (fChar - errorStart), + errorStart); + } + if (!this->skipExact("],\n")) { + return false; + } + if (!this->skipExact(" \"runtime_error\": \"")) { + return false; + } + if ('"' != this->peek()) { + const char* errorStart = fChar; + if (!this->skipToEndBracket('"')) { + return false; + } + SkDebugf("fiddle runtime error in %s: %.*s\n", name.c_str(), (int) (fChar - errorStart), + errorStart); + } + if (!this->skipExact("\",\n")) { + return false; + } + if (!this->skipExact(" \"fiddleHash\": \"")) { + return false; + } + const char* hashStart = fChar; + if (!this->skipToEndBracket('"')) { + return false; + } + Definition* example = this->findExample(name); + if (!example) { + SkDebugf("missing example %s\n", name.c_str()); + } + string hash(hashStart, fChar - hashStart); + if (example) { + example->fHash = hash; + } + if (!this->skipExact("\",\n")) { + return false; + } + if (!this->skipExact(" \"text\": \"")) { + return false; + } + if ('"' != this->peek()) { + const char* stdOutStart = fChar; + do { + if ('\\' == this->peek()) { + this->next(); + } else if ('"' == this->peek()) { + break; + } + } while (!this->eof() && this->next()); + const char* stdOutEnd = fChar; + if (example) { + bool foundStdOut = false; + for (auto& textOut : example->fChildren) { + if (MarkType::kStdOut != textOut->fMarkType) { + continue; + } + foundStdOut = true; + bool foundVolatile = false; + for (auto& stdOutChild : textOut->fChildren) { + if (MarkType::kVolatile == stdOutChild->fMarkType) { + foundVolatile = true; + break; + } + } + TextParser bmh(textOut); + EscapeParser fiddle(stdOutStart, stdOutEnd); + do { + bmh.skipWhiteSpace(); + fiddle.skipWhiteSpace(); + const char* bmhEnd = bmh.trimmedLineEnd(); + const char* fiddleEnd = fiddle.trimmedLineEnd(); + ptrdiff_t bmhLen = bmhEnd - bmh.fChar; + SkASSERT(bmhLen > 0); + ptrdiff_t fiddleLen = fiddleEnd - fiddle.fChar; + SkASSERT(fiddleLen > 0); + if (bmhLen != fiddleLen) { + if (!foundVolatile) { + SkDebugf("mismatched stdout len in %s\n", name.c_str()); + } + } else if (strncmp(bmh.fChar, fiddle.fChar, fiddleLen)) { + if (!foundVolatile) { + SkDebugf("mismatched stdout text in %s\n", name.c_str()); + } + } + bmh.skipToLineStart(); + fiddle.skipToLineStart(); + } while (!bmh.eof() && !fiddle.eof()); + if (!foundStdOut) { + SkDebugf("bmh %s missing stdout\n", name.c_str()); + } else if (!bmh.eof() || !fiddle.eof()) { + if (!foundVolatile) { + SkDebugf("%s mismatched stdout eof\n", name.c_str()); + } + } + } + } + } + if (!this->skipExact("\"\n")) { + return false; + } + if (!this->skipExact(" }")) { + return false; + } + if ('\n' == this->peek()) { + break; + } + if (!this->skipExact(",\n")) { + return false; + } + } +#if 0 + // compare the text output with the expected output in the markup tree + this->skipToSpace(); + SkASSERT(' ' == fChar[0]); + this->next(); + const char* nameLoc = fChar; + this->skipToNonAlphaNum(); + const char* nameEnd = fChar; + string name(nameLoc, nameEnd - nameLoc); + const Definition* example = this->findExample(name); + if (!example) { + return this->reportError<bool>("missing stdout name"); + } + SkASSERT(':' == fChar[0]); + this->next(); + this->skipSpace(); + const char* stdOutLoc = fChar; + do { + this->skipToLineStart(); + } while (!this->eof() && !this->startsWith("fiddles.htm:")); + const char* stdOutEnd = fChar; + for (auto& textOut : example->fChildren) { + if (MarkType::kStdOut != textOut->fMarkType) { + continue; + } + TextParser bmh(textOut); + TextParser fiddle(fFileName, stdOutLoc, stdOutEnd, fLineCount); + do { + bmh.skipWhiteSpace(); + fiddle.skipWhiteSpace(); + const char* bmhEnd = bmh.trimmedLineEnd(); + const char* fiddleEnd = fiddle.trimmedLineEnd(); + ptrdiff_t bmhLen = bmhEnd - bmh.fChar; + SkASSERT(bmhLen > 0); + ptrdiff_t fiddleLen = fiddleEnd - fiddle.fChar; + SkASSERT(fiddleLen > 0); + if (bmhLen != fiddleLen) { + return this->reportError<bool>("mismatched stdout len"); + } + if (strncmp(bmh.fChar, fiddle.fChar, fiddleLen)) { + return this->reportError<bool>("mismatched stdout text"); + } + bmh.skipToLineStart(); + fiddle.skipToLineStart(); + } while (!bmh.eof() && !fiddle.eof()); + if (!bmh.eof() || (!fiddle.eof() && !fiddle.startsWith("</pre>"))) { + return this->reportError<bool>("mismatched stdout eof"); + } + break; + } + } + } +#endif + return true; +} diff --git a/tools/bookmaker/includeParser.cpp b/tools/bookmaker/includeParser.cpp new file mode 100644 index 0000000000..089dcb3658 --- /dev/null +++ b/tools/bookmaker/includeParser.cpp @@ -0,0 +1,1733 @@ +/* + * Copyright 2017 Google Inc. + * + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include "bookmaker.h" + +enum class KeyProperty { + kNone, + kClassSection, + kFunction, + kModifier, + kNumber, + kObject, + kPreprocessor, +}; + +struct IncludeKey { + const char* fName; + KeyWord fKeyWord; + KeyProperty fProperty; +}; + +const IncludeKey kKeyWords[] = { + { "", KeyWord::kNone, KeyProperty::kNone }, + { "bool", KeyWord::kBool, KeyProperty::kNumber }, + { "char", KeyWord::kChar, KeyProperty::kNumber }, + { "class", KeyWord::kClass, KeyProperty::kObject }, + { "const", KeyWord::kConst, KeyProperty::kModifier }, + { "constexpr", KeyWord::kConstExpr, KeyProperty::kModifier }, + { "define", KeyWord::kDefine, KeyProperty::kPreprocessor }, + { "double", KeyWord::kDouble, KeyProperty::kNumber }, + { "elif", KeyWord::kElif, KeyProperty::kPreprocessor }, + { "else", KeyWord::kElse, KeyProperty::kPreprocessor }, + { "endif", KeyWord::kEndif, KeyProperty::kPreprocessor }, + { "enum", KeyWord::kEnum, KeyProperty::kObject }, + { "float", KeyWord::kFloat, KeyProperty::kNumber }, + { "friend", KeyWord::kFriend, KeyProperty::kModifier }, + { "if", KeyWord::kIf, KeyProperty::kPreprocessor }, + { "ifdef", KeyWord::kIfdef, KeyProperty::kPreprocessor }, + { "ifndef", KeyWord::kIfndef, KeyProperty::kPreprocessor }, + { "include", KeyWord::kInclude, KeyProperty::kPreprocessor }, + { "inline", KeyWord::kInline, KeyProperty::kModifier }, + { "int", KeyWord::kInt, KeyProperty::kNumber }, + { "operator", KeyWord::kOperator, KeyProperty::kFunction }, + { "private", KeyWord::kPrivate, KeyProperty::kClassSection }, + { "protected", KeyWord::kProtected, KeyProperty::kClassSection }, + { "public", KeyWord::kPublic, KeyProperty::kClassSection }, + { "signed", KeyWord::kSigned, KeyProperty::kNumber }, + { "size_t", KeyWord::kSize_t, KeyProperty::kNumber }, + { "static", KeyWord::kStatic, KeyProperty::kModifier }, + { "struct", KeyWord::kStruct, KeyProperty::kObject }, + { "template", KeyWord::kTemplate, KeyProperty::kObject }, + { "typedef", KeyWord::kTypedef, KeyProperty::kObject }, + { "uint32_t", KeyWord::kUint32_t, KeyProperty::kNumber }, + { "union", KeyWord::kUnion, KeyProperty::kObject }, + { "unsigned", KeyWord::kUnsigned, KeyProperty::kNumber }, + { "void", KeyWord::kVoid, KeyProperty::kNumber }, +}; + +const size_t kKeyWordCount = SK_ARRAY_COUNT(kKeyWords); + +KeyWord IncludeParser::FindKey(const char* start, const char* end) { + int ch = 0; + for (size_t index = 0; index < kKeyWordCount; ) { + if (start[ch] > kKeyWords[index].fName[ch]) { + ++index; + if (ch > 0 && kKeyWords[index - 1].fName[ch - 1] < kKeyWords[index].fName[ch - 1]) { + return KeyWord::kNone; + } + continue; + } + if (start[ch] < kKeyWords[index].fName[ch]) { + return KeyWord::kNone; + } + ++ch; + if (start + ch >= end) { + return kKeyWords[index].fKeyWord; + } + } + return KeyWord::kNone; +} + +void IncludeParser::ValidateKeyWords() { + for (size_t index = 1; index < kKeyWordCount; ++index) { + SkASSERT((int) kKeyWords[index - 1].fKeyWord + 1 + == (int) kKeyWords[index].fKeyWord); + SkASSERT(0 > strcmp(kKeyWords[index - 1].fName, kKeyWords[index].fName)); + } +} + +void IncludeParser::addKeyword(KeyWord keyWord) { + fParent->fTokens.emplace_back(keyWord, fIncludeWord, fChar, fLineCount, fParent); + fIncludeWord = nullptr; + if (KeyProperty::kObject == kKeyWords[(int) keyWord].fProperty) { + Definition* def = &fParent->fTokens.back(); + this->addDefinition(def); + if (KeyWord::kEnum == fParent->fKeyWord) { + fInEnum = true; + } + } +} + +void IncludeParser::checkForMissingParams(const vector<string>& methodParams, + const vector<string>& foundParams) { + for (auto& methodParam : methodParams) { + bool found = false; + for (auto& foundParam : foundParams) { + if (methodParam == foundParam) { + found = true; + break; + } + } + if (!found) { + this->keywordStart("Param"); + fprintf(fOut, "%s ", methodParam.c_str()); + this->keywordEnd(); + } + } + for (auto& foundParam : foundParams) { + bool found = false; + for (auto& methodParam : methodParams) { + if (methodParam == foundParam) { + found = true; + break; + } + } + if (!found) { + this->reportError("doxygen param does not match method declaration"); + } + } +} + +bool IncludeParser::checkForWord() { + if (!fIncludeWord) { + return true; + } + KeyWord keyWord = FindKey(fIncludeWord, fChar); + if (KeyWord::kNone != keyWord) { + if (KeyProperty::kPreprocessor != kKeyWords[(int) keyWord].fProperty) { + this->addKeyword(keyWord); + return true; + } + } else { + this->addWord(); + return true; + } + Definition* poundDef = fParent; + if (!fParent) { + return reportError<bool>("expected parent"); + } + if (Definition::Type::kBracket != poundDef->fType) { + return reportError<bool>("expected bracket"); + } + if (Bracket::kPound != poundDef->fBracket) { + return reportError<bool>("expected preprocessor"); + } + if (KeyWord::kNone != poundDef->fKeyWord) { + return reportError<bool>("already found keyword"); + } + poundDef->fKeyWord = keyWord; + fIncludeWord = nullptr; + switch (keyWord) { + // these do not link to other # directives + case KeyWord::kDefine: + case KeyWord::kInclude: + break; + // these start a # directive link + case KeyWord::kIf: + case KeyWord::kIfdef: + case KeyWord::kIfndef: + break; + // these continue a # directive link + case KeyWord::kElif: + case KeyWord::kElse: { + this->popObject(); // pop elif + if (Bracket::kPound != fParent->fBracket) { + return this->reportError<bool>("expected preprocessor directive"); + } + this->popBracket(); // pop if + poundDef->fParent = fParent; + this->addDefinition(poundDef); // push elif back + } break; + // this ends a # directive link + case KeyWord::kEndif: + // FIXME : should this be calling popBracket() instead? + this->popObject(); // pop endif + if (Bracket::kPound != fParent->fBracket) { + return this->reportError<bool>("expected preprocessor directive"); + } + this->popBracket(); // pop if/else + break; + default: + SkASSERT(0); + } + return true; +} + +string IncludeParser::className() const { + string name(fParent->fName); + size_t slash = name.find_last_of("/"); + if (string::npos == slash) { + slash = name.find_last_of("\\"); + } + SkASSERT(string::npos != slash); + string result = name.substr(slash); + result = result.substr(1, result.size() - 3); + return result; +} + +bool IncludeParser::crossCheck(BmhParser& bmhParser) { + string className = this->className(); + string classPrefix = className + "::"; + RootDefinition* root = &bmhParser.fClassMap[className]; + root->clearVisited(); + for (auto& classMapper : fIClassMap) { + if (className != classMapper.first + && classPrefix != classMapper.first.substr(0, classPrefix.length())) { + continue; + } + auto& classMap = classMapper.second; + auto& tokens = classMap.fTokens; + for (const auto& token : tokens) { + if (token.fPrivate) { + continue; + } + string fullName = classMapper.first + "::" + token.fName; + const Definition* def = root->find(fullName); + switch (token.fMarkType) { + case MarkType::kMethod: { + if (0 == token.fName.find("internal_") + || 0 == token.fName.find("Internal_") + || 0 == token.fName.find("legacy_") + || 0 == token.fName.find("temporary_")) { + continue; + } + const char* methodID = bmhParser.fMaps[(int) token.fMarkType].fName; + if (!def) { + string paramName = className + "::"; + paramName += string(token.fContentStart, + token.fContentEnd - token.fContentStart); + def = root->find(paramName); + if (!def && 0 == token.fName.find("operator")) { + string operatorName = className + "::"; + TextParser oper("", token.fStart, token.fContentEnd, 0); + const char* start = oper.strnstr("operator", token.fContentEnd); + SkASSERT(start); + oper.skipTo(start); + oper.skipToEndBracket('('); + int parens = 0; + do { + if ('(' == oper.peek()) { + ++parens; + } else if (')' == oper.peek()) { + --parens; + } + } while (!oper.eof() && oper.next() && parens > 0); + operatorName += string(start, oper.fChar - start); + def = root->find(operatorName); + } + } + if (!def) { + int skip = !strncmp(token.fContentStart, "explicit ", 9) ? 9 : 0; + skip = !strncmp(token.fContentStart, "virtual ", 8) ? 8 : skip; + string constructorName = className + "::"; + constructorName += string(token.fContentStart + skip, + token.fContentEnd - token.fContentStart - skip); + def = root->find(constructorName); + } + if (!def && 0 == token.fName.find("SK_")) { + string incName = token.fName + "()"; + string macroName = className + "::" + incName; + def = root->find(macroName); + if (def) { + if (def->fName == incName) { + def->fVisited = true; + if ("SK_TO_STRING_NONVIRT" == token.fName) { + def = root->find(className + "::toString"); + if (def) { + def->fVisited = true; + } else { + SkDebugf("missing toString bmh: %s\n", fullName.c_str()); + } + } + break; + } else { + SkDebugf("method macro differs from bmh: %s\n", fullName.c_str()); + } + } + } + if (!def) { + bool allLower = true; + for (size_t index = 0; index < token.fName.length(); ++index) { + if (!islower(token.fName[index])) { + allLower = false; + break; + } + } + if (allLower) { + string lowerName = className + "::" + token.fName + "()"; + def = root->find(lowerName); + } + } + if (!def) { + SkDebugf("method missing from bmh: %s\n", fullName.c_str()); + break; + } + if (def->crossCheck(methodID, token)) { + def->fVisited = true; + } else { + SkDebugf("method differs from bmh: %s\n", fullName.c_str()); + } + } break; + case MarkType::kComment: + break; + case MarkType::kEnum: { + if (!def) { + // work backwards from first word to deduce #Enum name + TextParser firstMember("", token.fStart, token.fContentEnd, 0); + SkAssertResult(firstMember.skipName("enum")); + SkAssertResult(firstMember.skipToEndBracket('{')); + firstMember.next(); + firstMember.skipWhiteSpace(); + SkASSERT('k' == firstMember.peek()); + const char* savePos = firstMember.fChar; + firstMember.skipToNonAlphaNum(); + const char* wordEnd = firstMember.fChar; + firstMember.fChar = savePos; + const char* lastUnderscore = nullptr; + do { + if (!firstMember.skipToEndBracket('_')) { + break; + } + if (firstMember.fChar > wordEnd) { + break; + } + lastUnderscore = firstMember.fChar; + } while (firstMember.next()); + if (lastUnderscore) { + ++lastUnderscore; + string anonName = className + "::" + string(lastUnderscore, + wordEnd - lastUnderscore) + 's'; + def = root->find(anonName); + } + if (!def) { + SkDebugf("enum missing from bmh: %s\n", fullName.c_str()); + break; + } + } + def->fVisited = true; + for (auto& child : def->fChildren) { + if (MarkType::kCode == child->fMarkType) { + def = child; + break; + } + } + if (MarkType::kCode != def->fMarkType) { + SkDebugf("enum code missing from bmh: %s\n", fullName.c_str()); + break; + } + if (def->crossCheck(token)) { + def->fVisited = true; + } else { + SkDebugf("enum differs from bmh: %s\n", def->fName.c_str()); + } + for (auto& child : token.fChildren) { + string constName = className + "::" + child->fName; + def = root->find(constName); + if (!def) { + string innerName = classMapper.first + "::" + child->fName; + def = root->find(innerName); + } + if (!def) { + if (string::npos == child->fName.find("Legacy_")) { + SkDebugf("const missing from bmh: %s\n", constName.c_str()); + } + } else { + def->fVisited = true; + } + } + } break; + case MarkType::kMember: + if (def) { + def->fVisited = true; + } else { + SkDebugf("member missing from bmh: %s\n", fullName.c_str()); + } + break; + default: + SkASSERT(0); // unhandled + break; + } + } + } + if (!root->dumpUnVisited()) { + SkDebugf("some struct elements not found; struct finding in includeParser is missing\n"); + } + return true; +} + +IClassDefinition* IncludeParser::defineClass(const Definition& includeDef, + const string& name) { + string className; + const Definition* test = fParent; + while (Definition::Type::kFileType != test->fType) { + if (Definition::Type::kMark == test->fType && KeyWord::kClass == test->fKeyWord) { + className = test->fName + "::"; + break; + } + test = test->fParent; + } + className += name; + unordered_map<string, IClassDefinition>& map = fIClassMap; + IClassDefinition& markupDef = map[className]; + if (markupDef.fStart) { + typedef IClassDefinition* IClassDefPtr; + return INHERITED::reportError<IClassDefPtr>("class already defined"); + } + markupDef.fFileName = fFileName; + markupDef.fStart = includeDef.fStart; + markupDef.fContentStart = includeDef.fStart; + markupDef.fName = className; + markupDef.fContentEnd = includeDef.fContentEnd; + markupDef.fTerminator = includeDef.fTerminator; + markupDef.fParent = fParent; + markupDef.fLineCount = fLineCount; + markupDef.fMarkType = KeyWord::kStruct == includeDef.fKeyWord ? + MarkType::kStruct : MarkType::kClass; + markupDef.fKeyWord = includeDef.fKeyWord; + markupDef.fType = Definition::Type::kMark; + fParent = &markupDef; + return &markupDef; +} + +void IncludeParser::dumpClassTokens(IClassDefinition& classDef) { + auto& tokens = classDef.fTokens; + for (auto& token : tokens) { + if (Definition::Type::kMark == token.fType && MarkType::kComment == token.fMarkType) { + continue; + } + if (MarkType::kMember != token.fMarkType) { + fprintf(fOut, "%s", + "# ------------------------------------------------------------------------------\n"); + fprintf(fOut, "" "\n"); + } + switch (token.fMarkType) { + case MarkType::kEnum: + fprintf(fOut, "#Enum %s" "\n", + token.fName.c_str()); + fprintf(fOut, "" "\n"); + fprintf(fOut, "#Code" "\n"); + fprintf(fOut, " enum %s {" "\n", + token.fName.c_str()); + for (auto& child : token.fChildren) { + fprintf(fOut, " %s %.*s" "\n", + child->fName.c_str(), child->length(), child->fContentStart); + } + fprintf(fOut, " };" "\n"); + fprintf(fOut, "##" "\n"); + fprintf(fOut, "" "\n"); + this->dumpComment(&token); + for (auto& child : token.fChildren) { + fprintf(fOut, "#Const %s", child->fName.c_str()); + TextParser val(child); + if (!val.eof()) { + if ('=' == val.fStart[0] || ',' == val.fStart[0]) { + val.next(); + val.skipSpace(); + const char* valEnd = val.anyOf(",\n"); + if (!valEnd) { + valEnd = val.fEnd; + } + fprintf(fOut, " %.*s", (int) (valEnd - val.fStart), val.fStart); + } else { + fprintf(fOut, " %.*s", + (int) (child->fContentEnd - child->fContentStart), + child->fContentStart); + } + } + fprintf(fOut, "" "\n"); + for (auto& token : child->fTokens) { + if (MarkType::kComment == token.fMarkType) { + this->dumpComment(&token); + } + } + fprintf(fOut, "##" "\n"); + } + fprintf(fOut, "" "\n"); + break; + case MarkType::kMethod: + fprintf(fOut, "#Method %.*s" "\n", + token.length(), token.fStart); + lfAlways(1); + this->dumpComment(&token); + break; + case MarkType::kMember: + this->keywordStart("Member"); + fprintf(fOut, "%.*s %s ", (int) (token.fContentEnd - token.fContentStart), + token.fContentStart, token.fName.c_str()); + lfAlways(1); + for (auto child : token.fChildren) { + fprintf(fOut, "%.*s", (int) (child->fContentEnd - child->fContentStart), + child->fContentStart); + lfAlways(1); + } + this->keywordEnd(); + continue; + break; + default: + SkASSERT(0); + } + this->lf(2); + fprintf(fOut, "#Example" "\n"); + fprintf(fOut, "##" "\n"); + fprintf(fOut, "" "\n"); + fprintf(fOut, "#ToDo incomplete ##" "\n"); + fprintf(fOut, "" "\n"); + fprintf(fOut, "##" "\n"); + fprintf(fOut, "" "\n"); + } +} +void IncludeParser::dumpComment(Definition* token) { + fLineCount = token->fLineCount; + fChar = fLine = token->fContentStart; + fEnd = token->fContentEnd; + bool sawParam = false; + bool multiline = false; + bool sawReturn = false; + bool sawComment = false; + bool methodHasReturn = false; + vector<string> methodParams; + vector<string> foundParams; + Definition methodName; + TextParser methodParser(token->fFileName, token->fContentStart, token->fContentEnd, + token->fLineCount); + if (MarkType::kMethod == token->fMarkType) { + methodName.fName = string(token->fContentStart, + (int) (token->fContentEnd - token->fContentStart)); + methodHasReturn = !methodParser.startsWith("void ") + && !methodParser.strnchr('~', methodParser.fEnd); + const char* paren = methodParser.strnchr('(', methodParser.fEnd); + const char* nextEnd = paren; + do { + string paramName; + methodParser.fChar = nextEnd + 1; + methodParser.skipSpace(); + if (!methodName.nextMethodParam(&methodParser, &nextEnd, ¶mName)) { + continue; + } + methodParams.push_back(paramName); + } while (')' != nextEnd[0]); + } + for (const auto& child : token->fTokens) { + if (Definition::Type::kMark == child.fType && MarkType::kComment == child.fMarkType) { + if ('@' == child.fContentStart[0]) { + TextParser parser(&child); + do { + parser.next(); + if (parser.startsWith("param ")) { + parser.skipWord("param"); + const char* parmStart = parser.fChar; + parser.skipToSpace(); + string parmName = string(parmStart, (int) (parser.fChar - parmStart)); + parser.skipWhiteSpace(); + do { + size_t nextComma = parmName.find(','); + string piece; + if (string::npos == nextComma) { + piece = parmName; + parmName = ""; + } else { + piece = parmName.substr(0, nextComma); + parmName = parmName.substr(nextComma + 1); + } + if (sawParam) { + if (multiline) { + this->lfAlways(1); + } + this->keywordEnd(); + } else { + if (sawComment) { + this->nl(); + } + this->lf(2); + } + foundParams.emplace_back(piece); + this->keywordStart("Param"); + fprintf(fOut, "%s ", piece.c_str()); + fprintf(fOut, "%.*s", (int) (parser.fEnd - parser.fChar), parser.fChar); + this->lfAlways(1); + sawParam = true; + sawComment = false; + } while (parmName.length()); + parser.skipTo(parser.fEnd); + } else if (parser.startsWith("return ") || parser.startsWith("returns ")) { + parser.skipWord("return"); + if ('s' == parser.peek()) { + parser.next(); + } + if (sawParam) { + if (multiline) { + this->lfAlways(1); + } + this->keywordEnd(); + } + this->checkForMissingParams(methodParams, foundParams); + sawParam = false; + sawComment = false; + multiline = false; + this->lf(2); + this->keywordStart("Return"); + fprintf(fOut, "%.*s ", (int) (parser.fEnd - parser.fChar), + parser.fChar); + this->lfAlways(1); + sawReturn = true; + parser.skipTo(parser.fEnd); + } else { + this->reportError("unexpected doxygen directive"); + } + } while (!parser.eof()); + } else { + if (sawComment) { + this->nl(); + } + this->lf(1); + fprintf(fOut, "%.*s ", child.length(), child.fContentStart); + sawComment = true; + if (sawParam || sawReturn) { + multiline = true; + } + } + } + } + if (sawParam || sawReturn) { + if (multiline) { + this->lfAlways(1); + } + this->keywordEnd(); + } + if (!sawReturn) { + if (!sawParam) { + if (sawComment) { + this->nl(); + } + this->lf(2); + } + this->checkForMissingParams(methodParams, foundParams); + } + if (methodHasReturn != sawReturn) { + if (!methodHasReturn) { + this->reportError("unexpected doxygen return"); + } else { + if (sawComment) { + this->nl(); + } + this->lf(2); + this->keywordStart("Return"); + this->keywordEnd(); + } + } +} + + // dump equivalent markup +void IncludeParser::dumpTokens() { + string skClassName = this->className(); + string fileName = skClassName + ".bmh"; + fOut = fopen(fileName.c_str(), "wb"); + if (!fOut) { + SkDebugf("could not open output file %s\n", fileName.c_str()); + return; + } + string prefixName = skClassName.substr(0, 2); + string topicName = skClassName.length() > 2 && isupper(skClassName[2]) && + ("Sk" == prefixName || "Gr" == prefixName) ? skClassName.substr(2) : skClassName; + fprintf(fOut, "#Topic %s", topicName.c_str()); + this->lfAlways(2); + fprintf(fOut, "#Class %s", skClassName.c_str()); + this->lfAlways(2); + auto& classMap = fIClassMap[skClassName]; + auto& tokens = classMap.fTokens; + for (auto& token : tokens) { + if (Definition::Type::kMark != token.fType || MarkType::kComment != token.fMarkType) { + continue; + } + fprintf(fOut, "%.*s", (int) (token.fContentEnd - token.fContentStart), + token.fContentStart); + this->lfAlways(1); + } + this->lf(2); + string className(skClassName.substr(2)); + vector<string> sortedClasses; + size_t maxLen = 0; + for (const auto& oneClass : fIClassMap) { + if (skClassName + "::" != oneClass.first.substr(0, skClassName.length() + 2)) { + continue; + } + string structName = oneClass.first.substr(skClassName.length() + 2); + maxLen = SkTMax(maxLen, structName.length()); + sortedClasses.emplace_back(structName); + } + fprintf(fOut, "#Topic Overview"); + this->lfAlways(2); + fprintf(fOut, "#Subtopic %s_Structs", className.c_str()); + this->lfAlways(1); + fprintf(fOut, "#Table"); + this->lfAlways(1); + fprintf(fOut, "#Legend"); + this->lfAlways(1); + fprintf(fOut, "# %-*s # description ##", (int) maxLen, "struct"); + this->lfAlways(1); + fprintf(fOut, "#Legend ##"); + this->lfAlways(1); + fprintf(fOut, "#Table ##"); + this->lfAlways(1); + for (auto& name : sortedClasses) { + fprintf(fOut, "# %-*s # ##", (int) maxLen, name.c_str()); + this->lfAlways(1); + } + fprintf(fOut, "#Subtopic ##"); + this->lfAlways(2); + fprintf(fOut, "#Subtopic %s_Member_Functions", className.c_str()); + this->lfAlways(1); + fprintf(fOut, "#Table"); + this->lfAlways(1); + fprintf(fOut, "#Legend"); + this->lfAlways(1); + maxLen = 0; + vector<string> sortedNames; + for (const auto& token : classMap.fTokens) { + if (Definition::Type::kMark != token.fType || MarkType::kMethod != token.fMarkType) { + continue; + } + const string& name = token.fName; + if (name.substr(0, 7) == "android" || string::npos != name.find("nternal_")) { + continue; + } + if (name[name.length() - 2] == '_' && isdigit(name[name.length() - 1])) { + continue; + } + size_t paren = name.find('('); + size_t funcLen = string::npos == paren ? name.length() : paren; + maxLen = SkTMax(maxLen, funcLen); + sortedNames.emplace_back(name); + } + std::sort(sortedNames.begin(), sortedNames.end()); + fprintf(fOut, "# %-*s # description ##" "\n", + (int) maxLen, "function"); + fprintf(fOut, "#Legend ##" "\n"); + for (auto& name : sortedNames) { + size_t paren = name.find('('); + size_t funcLen = string::npos == paren ? name.length() : paren; + fprintf(fOut, "# %-*s # ##" "\n", + (int) maxLen, name.substr(0, funcLen).c_str()); + } + fprintf(fOut, "#Table ##" "\n"); + fprintf(fOut, "#Subtopic ##" "\n"); + fprintf(fOut, "" "\n"); + fprintf(fOut, "#Topic ##" "\n"); + fprintf(fOut, "" "\n"); + + for (auto& oneClass : fIClassMap) { + if (skClassName + "::" != oneClass.first.substr(0, skClassName.length() + 2)) { + continue; + } + string innerName = oneClass.first.substr(skClassName.length() + 2); + fprintf(fOut, "%s", + "# ------------------------------------------------------------------------------"); + this->lfAlways(2); + fprintf(fOut, "#Struct %s", innerName.c_str()); + this->lfAlways(2); + for (auto& token : oneClass.second.fTokens) { + if (Definition::Type::kMark != token.fType || MarkType::kComment != token.fMarkType) { + continue; + } + fprintf(fOut, "%.*s", (int) (token.fContentEnd - token.fContentStart), + token.fContentStart); + this->lfAlways(1); + } + this->lf(2); + this->dumpClassTokens(oneClass.second); + this->lf(2); + fprintf(fOut, "#Struct %s ##", innerName.c_str()); + this->lfAlways(2); + } + this->dumpClassTokens(classMap); + fprintf(fOut, "#Class %s ##" "\n", + skClassName.c_str()); + fprintf(fOut, "" "\n"); + fprintf(fOut, "#Topic %s ##" "\n", + topicName.c_str()); + fclose(fOut); +} + +bool IncludeParser::findComments(const Definition& includeDef, Definition* markupDef) { + // add comment preceding class, if any + const Definition* parent = includeDef.fParent; + int index = includeDef.fParentIndex; + auto wordIter = parent->fTokens.begin(); + std::advance(wordIter, index); + SkASSERT(&*wordIter == &includeDef); + while (parent->fTokens.begin() != wordIter) { + auto testIter = std::prev(wordIter); + if (Definition::Type::kWord != testIter->fType + && Definition::Type::kKeyWord != testIter->fType + && (Definition::Type::kBracket != testIter->fType + || Bracket::kAngle != testIter->fBracket) + && (Definition::Type::kPunctuation != testIter->fType + || Punctuation::kAsterisk != testIter->fPunctuation)) { + break; + } + wordIter = testIter; + } + auto commentIter = wordIter; + while (parent->fTokens.begin() != commentIter) { + auto testIter = std::prev(commentIter); + bool isComment = Definition::Type::kBracket == testIter->fType + && (Bracket::kSlashSlash == testIter->fBracket + || Bracket::kSlashStar == testIter->fBracket); + if (!isComment) { + break; + } + commentIter = testIter; + } + while (commentIter != wordIter) { + if (!this->parseComment(commentIter->fFileName, commentIter->fContentStart, + commentIter->fContentEnd, commentIter->fLineCount, markupDef)) { + return false; + } + commentIter = std::next(commentIter); + } + return true; +} + +// caller calls reportError, so just return false here +bool IncludeParser::parseClass(Definition* includeDef, IsStruct isStruct) { + SkASSERT(includeDef->fTokens.size() > 0); + if (includeDef->fTokens.size() == 1) { + return true; // forward declaration only + } + // parse class header + auto iter = includeDef->fTokens.begin(); + if (!strncmp(iter->fStart, "SK_API", iter->fContentEnd - iter->fStart)) { + // todo : documentation is ignoring this for now + iter = std::next(iter); + } + string nameStr(iter->fStart, iter->fContentEnd - iter->fStart); + includeDef->fName = nameStr; + do { + if (iter == includeDef->fTokens.end()) { + return false; + } + if ('{' == iter->fStart[0] && Definition::Type::kPunctuation == iter->fType) { + break; + } + } while (static_cast<void>(iter = std::next(iter)), true); + if (Punctuation::kLeftBrace != iter->fPunctuation) { + return false; + } + IClassDefinition* markupDef = this->defineClass(*includeDef, nameStr); + if (!markupDef) { + return false; + } + markupDef->fStart = iter->fStart; + if (!this->findComments(*includeDef, markupDef)) { + return false; + } +// if (1 != includeDef->fChildren.size()) { +// return false; // fix me: SkCanvasClipVisitor isn't correctly parsed +// } + includeDef = includeDef->fChildren.front(); + iter = includeDef->fTokens.begin(); + // skip until public + int publicIndex = 0; + if (IsStruct::kNo == isStruct) { + const char* publicName = kKeyWords[(int) KeyWord::kPublic].fName; + size_t publicLen = strlen(publicName); + while (iter != includeDef->fTokens.end() + && (publicLen != (size_t) (iter->fContentEnd - iter->fStart) + || strncmp(iter->fStart, publicName, publicLen))) { + iter = std::next(iter); + ++publicIndex; + } + } + auto childIter = includeDef->fChildren.begin(); + while (childIter != includeDef->fChildren.end() && (*childIter)->fParentIndex < publicIndex) { + (*childIter)->fPrivate = true; + childIter = std::next(childIter); + } + int lastPublic = publicIndex; + const char* protectedName = kKeyWords[(int) KeyWord::kProtected].fName; + size_t protectedLen = strlen(protectedName); + const char* privateName = kKeyWords[(int) KeyWord::kPrivate].fName; + size_t privateLen = strlen(privateName); + while (iter != includeDef->fTokens.end() + && (protectedLen != (size_t) (iter->fContentEnd - iter->fStart) + || strncmp(iter->fStart, protectedName, protectedLen)) + && (privateLen != (size_t) (iter->fContentEnd - iter->fStart) + || strncmp(iter->fStart, privateName, privateLen))) { + iter = std::next(iter); + ++lastPublic; + } + while (childIter != includeDef->fChildren.end() && (*childIter)->fParentIndex < lastPublic) { + Definition* child = *childIter; + if (!this->parseObject(child, markupDef)) { + return false; + } + childIter = std::next(childIter); + } + while (childIter != includeDef->fChildren.end()) { + (*childIter)->fPrivate = true; + childIter = std::next(childIter); + } + SkASSERT(fParent->fParent); + fParent = fParent->fParent; + return true; +} + +bool IncludeParser::parseComment(const string& filename, const char* start, const char* end, + int lineCount, Definition* markupDef) { + TextParser parser(filename, start, end, lineCount); + // parse doxygen if present + if (parser.startsWith("**")) { + parser.next(); + parser.next(); + parser.skipWhiteSpace(); + if ('\\' == parser.peek()) { + parser.next(); + if (!parser.skipWord(kKeyWords[(int) markupDef->fKeyWord].fName)) { + return reportError<bool>("missing object type"); + } + if (!parser.skipWord(markupDef->fName.c_str())) { + return reportError<bool>("missing object name"); + } + + } + } + // remove leading '*' if present + Definition* parent = markupDef->fTokens.size() ? &markupDef->fTokens.back() : markupDef; + while (!parser.eof() && parser.skipWhiteSpace()) { + while ('*' == parser.peek()) { + parser.next(); + if (parser.eof()) { + break; + } + parser.skipWhiteSpace(); + } + if (parser.eof()) { + break; + } + const char* lineEnd = parser.trimmedLineEnd(); + markupDef->fTokens.emplace_back(MarkType::kComment, parser.fChar, lineEnd, + parser.fLineCount, parent); + parser.skipToEndBracket('\n'); + } + return true; +} + +bool IncludeParser::parseDefine() { + + return true; +} + +bool IncludeParser::parseEnum(Definition* child, Definition* markupDef) { + string nameStr; + if (child->fTokens.size() > 0) { + auto token = child->fTokens.begin(); + if (Definition::Type::kKeyWord == token->fType && KeyWord::kClass == token->fKeyWord) { + token = token->fTokens.begin(); + } + if (Definition::Type::kWord == token->fType) { + nameStr += string(token->fStart, token->fContentEnd - token->fStart); + } + } + markupDef->fTokens.emplace_back(MarkType::kEnum, child->fContentStart, child->fContentEnd, + child->fLineCount, markupDef); + Definition* markupChild = &markupDef->fTokens.back(); + SkASSERT(KeyWord::kNone == markupChild->fKeyWord); + markupChild->fKeyWord = KeyWord::kEnum; + TextParser enumName(child); + enumName.skipExact("enum "); + const char* nameStart = enumName.fChar; + enumName.skipToSpace(); + markupChild->fName = markupDef->fName + "::" + + string(nameStart, (size_t) (enumName.fChar - nameStart)); + if (!this->findComments(*child, markupChild)) { + return false; + } + TextParser parser(child); + parser.skipToEndBracket('{'); + const char* dataEnd; + do { + parser.next(); + parser.skipWhiteSpace(); + if ('}' == parser.peek()) { + break; + } + Definition* comment = nullptr; + // note that comment, if any, can be before or after (on the same line, though) as member + if ('#' == parser.peek()) { + // fixme: handle preprecessor, but just skip it for now + parser.skipToLineStart(); + } + while (parser.startsWith("/*") || parser.startsWith("//")) { + parser.next(); + const char* start = parser.fChar; + const char* end; + if ('*' == parser.peek()) { + end = parser.strnstr("*/", parser.fEnd); + parser.fChar = end; + parser.next(); + parser.next(); + } else { + end = parser.trimmedLineEnd(); + parser.skipToLineStart(); + } + markupChild->fTokens.emplace_back(MarkType::kComment, start, end, parser.fLineCount, + markupChild); + comment = &markupChild->fTokens.back(); + comment->fTerminator = end; + if (!this->parseComment(parser.fFileName, start, end, parser.fLineCount, comment)) { + return false; + } + parser.skipWhiteSpace(); + } + parser.skipWhiteSpace(); + const char* memberStart = parser.fChar; + if ('}' == memberStart[0]) { + break; + } + parser.skipToNonAlphaNum(); + string memberName(memberStart, parser.fChar); + parser.skipWhiteSpace(); + const char* dataStart = parser.fChar; + SkASSERT('=' == dataStart[0] || ',' == dataStart[0] || '}' == dataStart[0] + || '/' == dataStart[0]); + dataEnd = parser.anyOf(",}"); + markupChild->fTokens.emplace_back(MarkType::kMember, dataStart, dataEnd, parser.fLineCount, + markupChild); + Definition* member = &markupChild->fTokens.back(); + member->fName = memberName; + if (comment) { + member->fChildren.push_back(comment); + } + markupChild->fChildren.push_back(member); + parser.skipToEndBracket(dataEnd[0]); + } while (',' == dataEnd[0]); + for (size_t index = 1; index < child->fChildren.size(); ++index) { + const Definition* follower = child->fChildren[index]; + if (Definition::Type::kKeyWord == follower->fType) { + markupChild->fTokens.emplace_back(MarkType::kMember, follower->fContentStart, + follower->fContentEnd, follower->fLineCount, markupChild); + Definition* member = &markupChild->fTokens.back(); + member->fName = follower->fName; + markupChild->fChildren.push_back(member); + } + } + IClassDefinition& classDef = fIClassMap[markupDef->fName]; + SkASSERT(classDef.fStart); + string uniqueName = this->uniqueName(classDef.fEnums, nameStr); + markupChild->fName = uniqueName; + classDef.fEnums[uniqueName] = markupChild; + return true; +} + +bool IncludeParser::parseInclude(const string& name) { + fParent = &fIncludeMap[name]; + fParent->fName = name; + fParent->fFileName = fFileName; + fParent->fType = Definition::Type::kFileType; + fParent->fContentStart = fChar; + fParent->fContentEnd = fEnd; + // parse include file into tree + while (fChar < fEnd) { + if (!this->parseChar()) { + return false; + } + } + // parse tree and add named objects to maps + fParent = &fIncludeMap[name]; + if (!this->parseObjects(fParent, nullptr)) { + return false; + } + return true; +} + +bool IncludeParser::parseMember(Definition* child, Definition* markupDef) { + const char* typeStart = child->fChildren[0]->fContentStart; + markupDef->fTokens.emplace_back(MarkType::kMember, typeStart, child->fContentStart, + child->fLineCount, markupDef); + Definition* markupChild = &markupDef->fTokens.back(); + TextParser nameParser(child); + nameParser.skipToNonAlphaNum(); + string nameStr = string(child->fContentStart, nameParser.fChar - child->fContentStart); + IClassDefinition& classDef = fIClassMap[markupDef->fName]; + string uniqueName = this->uniqueName(classDef.fMethods, nameStr); + markupChild->fName = uniqueName; + classDef.fMembers[uniqueName] = markupChild; + if (child->fParentIndex >= 2) { + auto comment = child->fParent->fTokens.begin(); + std::advance(comment, child->fParentIndex - 2); + if (Definition::Type::kBracket == comment->fType + && (Bracket::kSlashStar == comment->fBracket + || Bracket::kSlashSlash == comment->fBracket)) { + TextParser parser(&*comment); + do { + parser.skipToAlpha(); + if (parser.eof()) { + break; + } + const char* start = parser.fChar; + const char* end = parser.trimmedBracketEnd('\n', OneLine::kYes); + if (Bracket::kSlashStar == comment->fBracket) { + const char* commentEnd = parser.strnstr("*/", end); + if (commentEnd) { + end = commentEnd; + } + } + markupDef->fTokens.emplace_back(MarkType::kComment, start, end, child->fLineCount, + markupDef); + Definition* commentChild = &markupDef->fTokens.back(); + markupChild->fChildren.emplace_back(commentChild); + parser.skipTo(end); + } while (!parser.eof()); + } + } + return true; +} + +bool IncludeParser::parseMethod(Definition* child, Definition* markupDef) { + auto tokenIter = child->fParent->fTokens.begin(); + std::advance(tokenIter, child->fParentIndex); + tokenIter = std::prev(tokenIter); + string nameStr(tokenIter->fStart, tokenIter->fContentEnd - tokenIter->fStart); + while (tokenIter != child->fParent->fTokens.begin()) { + auto testIter = std::prev(tokenIter); + switch (testIter->fType) { + case Definition::Type::kWord: + goto keepGoing; + case Definition::Type::kKeyWord: { + KeyProperty keyProperty = kKeyWords[(int) testIter->fKeyWord].fProperty; + if (KeyProperty::kNumber == keyProperty || KeyProperty::kModifier == keyProperty) { + goto keepGoing; + } + } break; + case Definition::Type::kBracket: + if (Bracket::kAngle == testIter->fBracket) { + goto keepGoing; + } + break; + case Definition::Type::kPunctuation: + if (Punctuation::kSemicolon == testIter->fPunctuation + || Punctuation::kLeftBrace == testIter->fPunctuation + || Punctuation::kColon == testIter->fPunctuation) { + break; + } + keepGoing: + tokenIter = testIter; + continue; + default: + break; + } + break; + } + tokenIter->fName = nameStr; + tokenIter->fMarkType = MarkType::kMethod; + auto testIter = child->fParent->fTokens.begin(); + SkASSERT(child->fParentIndex > 0); + std::advance(testIter, child->fParentIndex - 1); + const char* start = tokenIter->fContentStart; + const char* end = tokenIter->fContentEnd; + const char kDebugCodeStr[] = "SkDEBUGCODE"; + const size_t kDebugCodeLen = sizeof(kDebugCodeStr) - 1; + if (end - start == kDebugCodeLen && !strncmp(start, kDebugCodeStr, kDebugCodeLen)) { + std::advance(testIter, 1); + start = testIter->fContentStart + 1; + end = testIter->fContentEnd - 1; + } else { + end = testIter->fContentEnd; + while (testIter != child->fParent->fTokens.end()) { + testIter = std::next(testIter); + switch (testIter->fType) { + case Definition::Type::kPunctuation: + SkASSERT(Punctuation::kSemicolon == testIter->fPunctuation + || Punctuation::kLeftBrace == testIter->fPunctuation + || Punctuation::kColon == testIter->fPunctuation); + end = testIter->fStart; + break; + case Definition::Type::kKeyWord: { + KeyProperty keyProperty = kKeyWords[(int) testIter->fKeyWord].fProperty; + if (KeyProperty::kNumber == keyProperty || KeyProperty::kModifier == keyProperty) { + continue; + } + } break; + default: + continue; + } + break; + } + } + markupDef->fTokens.emplace_back(MarkType::kMethod, start, end, tokenIter->fLineCount, + markupDef); + Definition* markupChild = &markupDef->fTokens.back(); + // do find instead -- I wonder if there is a way to prevent this in c++ + IClassDefinition& classDef = fIClassMap[markupDef->fName]; + SkASSERT(classDef.fStart); + string uniqueName = this->uniqueName(classDef.fMethods, nameStr); + markupChild->fName = uniqueName; + if (!this->findComments(*child, markupChild)) { + return false; + } + classDef.fMethods[uniqueName] = markupChild; + return true; +} + +void IncludeParser::keywordEnd() { + fprintf(fOut, "##"); + this->lfAlways(1); +} + +void IncludeParser::keywordStart(const char* keyword) { + this->lf(1); + fprintf(fOut, "#%s ", keyword); +} + +bool IncludeParser::parseObjects(Definition* parent, Definition* markupDef) { + for (auto& child : parent->fChildren) { + if (!this->parseObject(child, markupDef)) { + return false; + } + } + return true; +} + +bool IncludeParser::parseObject(Definition* child, Definition* markupDef) { + // set up for error reporting + fLine = fChar = child->fStart; + fEnd = child->fContentEnd; + // todo: put original line number in child as well + switch (child->fType) { + case Definition::Type::kKeyWord: + switch (child->fKeyWord) { + case KeyWord::kClass: + if (!this->parseClass(child, IsStruct::kNo)) { + return this->reportError<bool>("failed to parse class"); + } + break; + case KeyWord::kEnum: + if (!this->parseEnum(child, markupDef)) { + return this->reportError<bool>("failed to parse enum"); + } + break; + case KeyWord::kStruct: + if (!this->parseClass(child, IsStruct::kYes)) { + return this->reportError<bool>("failed to parse struct"); + } + break; + case KeyWord::kTemplate: + if (!this->parseTemplate()) { + return this->reportError<bool>("failed to parse template"); + } + break; + case KeyWord::kTypedef: + if (!this->parseTypedef()) { + return this->reportError<bool>("failed to parse typedef"); + } + break; + case KeyWord::kUnion: + if (!this->parseUnion()) { + return this->reportError<bool>("failed to parse union"); + } + break; + default: + return this->reportError<bool>("unhandled keyword"); + } + break; + case Definition::Type::kBracket: + switch (child->fBracket) { + case Bracket::kParen: + if (!this->parseMethod(child, markupDef)) { + return this->reportError<bool>("failed to parse method"); + } + break; + case Bracket::kSlashSlash: + case Bracket::kSlashStar: + // comments are picked up by parsing objects first + break; + case Bracket::kPound: + // special-case the #xxx xxx_DEFINED entries + switch (child->fKeyWord) { + case KeyWord::kIfndef: + case KeyWord::kIfdef: + if (child->boilerplateIfDef(fParent)) { + if (!this->parseObjects(child, markupDef)) { + return false; + } + break; + } + goto preproError; + case KeyWord::kDefine: + if (child->boilerplateDef(fParent)) { + break; + } + goto preproError; + case KeyWord::kEndif: + if (child->boilerplateEndIf()) { + break; + } + case KeyWord::kInclude: + // ignored for now + break; + case KeyWord::kElse: + case KeyWord::kElif: + // todo: handle these + break; + default: + preproError: + return this->reportError<bool>("unhandled preprocessor"); + } + break; + case Bracket::kAngle: + // pick up templated function pieces when method is found + break; + default: + return this->reportError<bool>("unhandled bracket"); + } + break; + case Definition::Type::kWord: + if (MarkType::kMember != child->fMarkType) { + return this->reportError<bool>("unhandled word type"); + } + if (!this->parseMember(child, markupDef)) { + return this->reportError<bool>("unparsable member"); + } + break; + default: + return this->reportError<bool>("unhandled type"); + break; + } + return true; +} + +bool IncludeParser::parseTemplate() { + + return true; +} + +bool IncludeParser::parseTypedef() { + + return true; +} + +bool IncludeParser::parseUnion() { + + return true; +} + +bool IncludeParser::parseChar() { + char test = *fChar; + if ('\\' == fPrev) { + if ('\n' == test) { + ++fLineCount; + fLine = fChar + 1; + } + goto done; + } + switch (test) { + case '\n': + ++fLineCount; + fLine = fChar + 1; + if (fInChar) { + return reportError<bool>("malformed char"); + } + if (fInString) { + return reportError<bool>("malformed string"); + } + if (!this->checkForWord()) { + return false; + } + if (Bracket::kPound == this->topBracket()) { + KeyWord keyWord = fParent->fKeyWord; + if (KeyWord::kNone == keyWord) { + return this->reportError<bool>("unhandled preprocessor directive"); + } + if (KeyWord::kInclude == keyWord || KeyWord::kDefine == keyWord) { + this->popBracket(); + } + } else if (Bracket::kSlashSlash == this->topBracket()) { + this->popBracket(); + } + break; + case '*': + if (!fInCharCommentString && '/' == fPrev) { + this->pushBracket(Bracket::kSlashStar); + } + if (!this->checkForWord()) { + return false; + } + if (!fInCharCommentString) { + this->addPunctuation(Punctuation::kAsterisk); + } + break; + case '/': + if ('*' == fPrev) { + if (!fInCharCommentString) { + return reportError<bool>("malformed closing comment"); + } + if (Bracket::kSlashStar == this->topBracket()) { + this->popBracket(); + } + break; + } + if (!fInCharCommentString && '/' == fPrev) { + this->pushBracket(Bracket::kSlashSlash); + break; + } + if (!this->checkForWord()) { + return false; + } + break; + case '\'': + if (Bracket::kChar == this->topBracket()) { + this->popBracket(); + } else if (!fInComment && !fInString) { + if (fIncludeWord) { + return this->reportError<bool>("word then single-quote"); + } + this->pushBracket(Bracket::kChar); + } + break; + case '\"': + if (Bracket::kString == this->topBracket()) { + this->popBracket(); + } else if (!fInComment && !fInChar) { + if (fIncludeWord) { + return this->reportError<bool>("word then double-quote"); + } + this->pushBracket(Bracket::kString); + } + break; + case ':': + case '(': + case '[': + case '{': { + if (fInCharCommentString) { + break; + } + if (':' == test && (fInBrace || ':' == fChar[-1] || ':' == fChar[1])) { + break; + } + if (!fInBrace) { + if (!this->checkForWord()) { + return false; + } + if (':' == test && !fInFunction) { + break; + } + if ('{' == test) { + this->addPunctuation(Punctuation::kLeftBrace); + } else if (':' == test) { + this->addPunctuation(Punctuation::kColon); + } + } + if (fInBrace && '{' == test && Definition::Type::kBracket == fInBrace->fType + && Bracket::kColon == fInBrace->fBracket) { + Definition* braceParent = fParent->fParent; + braceParent->fChildren.pop_back(); + braceParent->fTokens.pop_back(); + fParent = braceParent; + fInBrace = nullptr; + } + this->pushBracket( + '(' == test ? Bracket::kParen : + '[' == test ? Bracket::kSquare : + '{' == test ? Bracket::kBrace : + Bracket::kColon); + if (!fInBrace + && ('{' == test || (':' == test && ' ' >= fChar[1])) + && fInFunction) { + fInBrace = fParent; + } + } break; + case '<': + if (fInCharCommentString || fInBrace) { + break; + } + if (!this->checkForWord()) { + return false; + } + if (fInEnum) { + break; + } + this->pushBracket(Bracket::kAngle); + break; + case ')': + case ']': + case '}': { + if (fInCharCommentString) { + break; + } + if (!fInBrace) { + if (!this->checkForWord()) { + return false; + } + } + bool popBraceParent = fInBrace == fParent; + if ((')' == test ? Bracket::kParen : + ']' == test ? Bracket::kSquare : Bracket::kBrace) == this->topBracket()) { + this->popBracket(); + if (!fInFunction) { + bool deprecatedMacro = false; + if (')' == test) { + auto iter = fParent->fTokens.end(); + bool lookForWord = false; + while (fParent->fTokens.begin() != iter) { + --iter; + if (lookForWord) { + if (Definition::Type::kWord != iter->fType) { + break; + } + string word(iter->fContentStart, iter->length()); + if ("SK_ATTR_EXTERNALLY_DEPRECATED" == word) { + deprecatedMacro = true; + // remove macro paren (would confuse method parsing later) + fParent->fTokens.pop_back(); + fParent->fChildren.pop_back(); + } + break; + } + if (Definition::Type::kBracket != iter->fType) { + break; + } + if (Bracket::kParen != iter->fBracket) { + break; + } + lookForWord = true; + } + } + fInFunction = ')' == test && !deprecatedMacro; + } else { + fInFunction = '}' != test; + } + } else { + return reportError<bool>("malformed close bracket"); + } + if (popBraceParent) { + Definition* braceParent = fInBrace->fParent; + braceParent->fChildren.pop_back(); + braceParent->fTokens.pop_back(); + fInBrace = nullptr; + } + } break; + case '>': + if (fInCharCommentString || fInBrace) { + break; + } + if (!this->checkForWord()) { + return false; + } + if (fInEnum) { + break; + } + if (Bracket::kAngle == this->topBracket()) { + this->popBracket(); + } else { + return reportError<bool>("malformed close angle bracket"); + } + break; + case '#': { + if (fInCharCommentString || fInBrace) { + break; + } + SkASSERT(!fIncludeWord); // don't expect this, curious if it is triggered + this->pushBracket(Bracket::kPound); + break; + } + case '&': + case ',': + case ' ': + if (fInCharCommentString || fInBrace) { + break; + } + if (!this->checkForWord()) { + return false; + } + break; + case ';': + if (fInCharCommentString || fInBrace) { + break; + } + if (!this->checkForWord()) { + return false; + } + if (Definition::Type::kKeyWord == fParent->fType + && KeyProperty::kObject == (kKeyWords[(int) fParent->fKeyWord].fProperty)) { + if (KeyWord::kEnum == fParent->fKeyWord) { + fInEnum = false; + } + this->popObject(); + } else if (Definition::Type::kBracket == fParent->fType + && fParent->fParent && Definition::Type::kKeyWord == fParent->fParent->fType + && KeyWord::kStruct == fParent->fParent->fKeyWord) { + list<Definition>::iterator baseIter = fParent->fTokens.end(); + list<Definition>::iterator namedIter = fParent->fTokens.end(); + for (auto tokenIter = fParent->fTokens.end(); + fParent->fTokens.begin() != tokenIter--; ) { + if (tokenIter->fLineCount == fLineCount) { + if ('f' == tokenIter->fStart[0] && isupper(tokenIter->fStart[1])) { + if (namedIter != fParent->fTokens.end()) { + return reportError<bool>("found two named member tokens"); + } + namedIter = tokenIter; + } + baseIter = tokenIter; + } else { + break; + } + } + // FIXME: if a member definition spans multiple lines, this won't work + if (namedIter != fParent->fTokens.end()) { + if (baseIter == namedIter) { + return this->reportError<bool>("expected type before named token"); + } + Definition* member = &*namedIter; + member->fMarkType = MarkType::kMember; + fParent->fChildren.push_back(member); + for (auto nameType = baseIter; nameType != namedIter; ++nameType) { + member->fChildren.push_back(&*nameType); + } + + } + } else if (fParent->fChildren.size() > 0) { + auto lastIter = fParent->fChildren.end(); + Definition* priorEnum; + while (fParent->fChildren.begin() != lastIter) { + std::advance(lastIter, -1); + priorEnum = *lastIter; + if (Definition::Type::kBracket != priorEnum->fType || + (Bracket::kSlashSlash != priorEnum->fBracket + && Bracket::kSlashStar != priorEnum->fBracket)) { + break; + } + } + if (Definition::Type::kKeyWord == priorEnum->fType + && KeyWord::kEnum == priorEnum->fKeyWord) { + auto tokenWalker = fParent->fTokens.begin(); + std::advance(tokenWalker, priorEnum->fParentIndex); + SkASSERT(KeyWord::kEnum == tokenWalker->fKeyWord); + while (tokenWalker != fParent->fTokens.end()) { + std::advance(tokenWalker, 1); + if (Punctuation::kSemicolon == tokenWalker->fPunctuation) { + break; + } + } + while (tokenWalker != fParent->fTokens.end()) { + std::advance(tokenWalker, 1); + const Definition* test = &*tokenWalker; + if (Definition::Type::kBracket != test->fType || + (Bracket::kSlashSlash != test->fBracket + && Bracket::kSlashStar != test->fBracket)) { + break; + } + } + Definition* start = &*tokenWalker; + bool foundExpected = true; + for (KeyWord expected : {KeyWord::kStatic, KeyWord::kConstExpr, KeyWord::kInt}){ + const Definition* test = &*tokenWalker; + if (expected != test->fKeyWord) { + foundExpected = false; + break; + } + if (tokenWalker == fParent->fTokens.end()) { + break; + } + std::advance(tokenWalker, 1); + } + if (foundExpected && tokenWalker != fParent->fTokens.end()) { + const char* nameStart = tokenWalker->fStart; + std::advance(tokenWalker, 1); + if (tokenWalker != fParent->fTokens.end()) { + TextParser tp(fFileName, nameStart, tokenWalker->fStart, fLineCount); + tp.skipToNonAlphaNum(); + start->fName = string(nameStart, tp.fChar - nameStart); + start->fContentEnd = fChar; + priorEnum->fChildren.emplace_back(start); + } + } + } + } + this->addPunctuation(Punctuation::kSemicolon); + fInFunction = false; + break; + case '~': + if (fInEnum) { + break; + } + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + // TODO: don't want to parse numbers, but do need to track for enum defs + // break; + case 'A': case 'B': case 'C': case 'D': case 'E': + case 'F': case 'G': case 'H': case 'I': case 'J': + case 'K': case 'L': case 'M': case 'N': case 'O': + case 'P': case 'Q': case 'R': case 'S': case 'T': + case 'U': case 'V': case 'W': case 'X': case 'Y': + case 'Z': case '_': + case 'a': case 'b': case 'c': case 'd': case 'e': + case 'f': case 'g': case 'h': case 'i': case 'j': + case 'k': case 'l': case 'm': case 'n': case 'o': + case 'p': case 'q': case 'r': case 's': case 't': + case 'u': case 'v': case 'w': case 'x': case 'y': + case 'z': + if (fInCharCommentString || fInBrace) { + break; + } + if (!fIncludeWord) { + fIncludeWord = fChar; + } + break; + } +done: + fPrev = test; + ++fChar; + return true; +} + +void IncludeParser::validate() const { + for (int index = 0; index <= (int) Last_MarkType; ++index) { + SkASSERT(fMaps[index].fMarkType == (MarkType) index); + } + IncludeParser::ValidateKeyWords(); +} diff --git a/tools/bookmaker/includeWriter.cpp b/tools/bookmaker/includeWriter.cpp new file mode 100644 index 0000000000..5685f31339 --- /dev/null +++ b/tools/bookmaker/includeWriter.cpp @@ -0,0 +1,1272 @@ +/* + * Copyright 2017 Google Inc. + * + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include "bookmaker.h" + +void IncludeWriter::enumHeaderOut(const RootDefinition* root, + const Definition& child) { + const Definition* enumDef = nullptr; + const char* bodyEnd = fDeferComment ? fDeferComment->fContentStart - 1 : + child.fContentStart; + this->writeBlockTrim((int) (bodyEnd - fStart), fStart); // may write nothing + this->lf(2); + fDeferComment = nullptr; + fStart = child.fContentStart; + const auto& nameDef = child.fTokens.front(); + string fullName; + if (nullptr != nameDef.fContentEnd) { + string enumName(nameDef.fContentStart, + (int) (nameDef.fContentEnd - nameDef.fContentStart)); + fullName = root->fName + "::" + enumName; + enumDef = root->find(enumName); + if (!enumDef) { + enumDef = root->find(fullName); + } + SkASSERT(enumDef); + // child[0] should be #Code comment starts at child[0].fTerminator + // though skip until #Code is found (in case there's a #ToDo, etc) + // child[1] should be #Const comment ends at child[1].fStart + // comment becomes enum header (if any) + } else { + string enumName(root->fName); + enumName += "::_anonymous"; + if (fAnonymousEnumCount > 1) { + enumName += '_' + to_string(fAnonymousEnumCount); + } + enumDef = root->find(enumName); + SkASSERT(enumDef); + ++fAnonymousEnumCount; + } + Definition* codeBlock = nullptr; + const char* commentStart = nullptr; + bool wroteHeader = false; + SkDEBUGCODE(bool foundConst = false); + for (auto test : enumDef->fChildren) { + if (MarkType::kCode == test->fMarkType) { + SkASSERT(!codeBlock); // FIXME: check enum for correct order earlier + codeBlock = test; + commentStart = codeBlock->fTerminator; + continue; + } + if (!codeBlock) { + continue; + } + const char* commentEnd = test->fStart; + if (!wroteHeader && + !this->contentFree((int) (commentEnd - commentStart), commentStart)) { + this->writeCommentHeader(); + this->writeString("\\enum"); + this->writeSpace(); + this->writeString(fullName.c_str()); + fIndent += 4; + this->lfcr(); + wroteHeader = true; + } + this->rewriteBlock((int) (commentEnd - commentStart), commentStart); + if (MarkType::kAnchor == test->fMarkType) { + commentStart = test->fContentStart; + commentEnd = test->fChildren[0]->fStart; + this->writeSpace(); + this->rewriteBlock((int) (commentEnd - commentStart), commentStart); + this->writeSpace(); + } + commentStart = test->fTerminator; + if (MarkType::kConst == test->fMarkType) { + SkASSERT(codeBlock); // FIXME: check enum for correct order earlier + SkDEBUGCODE(foundConst = true); + break; + } + } + SkASSERT(codeBlock); + SkASSERT(foundConst); + if (wroteHeader) { + fIndent -= 4; + this->lfcr(); + this->writeCommentTrailer(); + } + bodyEnd = child.fChildren[0]->fContentStart; + SkASSERT('{' == bodyEnd[0]); + ++bodyEnd; + this->lfcr(); + this->writeBlock((int) (bodyEnd - fStart), fStart); // write include "enum Name {" + fIndent += 4; + this->singleLF(); + fStart = bodyEnd; + fEnumDef = enumDef; +} + +void IncludeWriter::enumMembersOut(const RootDefinition* root, const Definition& child) { + // iterate through include tokens and find how much remains for 1 line comments + // put ones that fit on same line, ones that are too big on preceding line? + const Definition* currentEnumItem = nullptr; + const char* commentStart = nullptr; + const char* lastEnd = nullptr; + int commentLen = 0; + enum class State { + kNoItem, + kItemName, + kItemValue, + kItemComment, + }; + State state = State::kNoItem; + // can't use (auto& token : child.fTokens) 'cause we need state one past end + auto tokenIter = child.fTokens.begin(); + for (int onePast = 0; onePast < 2; onePast += tokenIter == child.fTokens.end()) { + const Definition* token = onePast ? nullptr : &*tokenIter++; + if (token && Definition::Type::kBracket == token->fType) { + if (Bracket::kSlashSlash == token->fBracket) { + fStart = token->fContentEnd; + continue; // ignore old inline comments + } + if (Bracket::kSlashStar == token->fBracket) { + fStart = token->fContentEnd + 1; + continue; // ignore old inline comments + } + SkASSERT(0); // incomplete + } + if (token && Definition::Type::kWord != token->fType) { + SkASSERT(0); // incomplete + } + if (token && State::kItemName == state) { + TextParser enumLine(token->fFileName, lastEnd, + token->fContentStart, token->fLineCount); + const char* end = enumLine.anyOf(",}="); + SkASSERT(end); + state = '=' == *end ? State::kItemValue : State::kItemComment; + if (State::kItemValue == state) { // write enum value + this->indentToColumn(fEnumItemValueTab); + this->writeString("="); + this->writeSpace(); + lastEnd = token->fContentEnd; + this->writeBlock((int) (lastEnd - token->fContentStart), + token->fContentStart); // write const value if any + continue; + } + } + if (token && State::kItemValue == state) { + TextParser valueEnd(token->fFileName, lastEnd, + token->fContentStart, token->fLineCount); + const char* end = valueEnd.anyOf(",}"); + if (!end) { // write expression continuation + if (' ' == lastEnd[0]) { + this->writeSpace(); + } + this->writeBlock((int) (token->fContentEnd - lastEnd), lastEnd); + continue; + } + } + if (State::kNoItem != state) { + this->writeString(","); + SkASSERT(currentEnumItem); + if (currentEnumItem->fShort) { + this->indentToColumn(fEnumItemCommentTab); + this->writeString("//!<"); + this->writeSpace(); + this->rewriteBlock(commentLen, commentStart); + } + if (onePast) { + fIndent -= 4; + } + this->lfcr(); + if (token && State::kItemValue == state) { + fStart = token->fContentStart; + } + state = State::kNoItem; + } + SkASSERT(State::kNoItem == state); + if (onePast) { + break; + } + SkASSERT(token); + string itemName = root->fName + "::" + string(token->fContentStart, + (int) (token->fContentEnd - token->fContentStart)); + for (auto& enumItem : fEnumDef->fChildren) { + if (MarkType::kConst != enumItem->fMarkType) { + continue; + } + if (itemName != enumItem->fName) { + continue; + } + currentEnumItem = enumItem; + break; + } + SkASSERT(currentEnumItem); + // if description fits, it goes after item + commentStart = currentEnumItem->fContentStart; + const char* commentEnd; + if (currentEnumItem->fChildren.size() > 0) { + commentEnd = currentEnumItem->fChildren[0]->fStart; + } else { + commentEnd = currentEnumItem->fContentEnd; + } + TextParser enumComment(fFileName, commentStart, commentEnd, currentEnumItem->fLineCount); + if (enumComment.skipToLineStart()) { // skip const value + commentStart = enumComment.fChar; + commentLen = (int) (commentEnd - commentStart); + } else { + const Definition* privateDef = currentEnumItem->fChildren[0]; + SkASSERT(MarkType::kPrivate == privateDef->fMarkType); + commentStart = privateDef->fContentStart; + commentLen = (int) (privateDef->fContentEnd - privateDef->fContentStart); + } + SkASSERT(commentLen > 0 && commentLen < 1000); + if (!currentEnumItem->fShort) { + this->writeCommentHeader(); + fIndent += 4; + bool wroteLineFeed = Wrote::kLF == this->rewriteBlock(commentLen, commentStart); + fIndent -= 4; + if (wroteLineFeed || fColumn > 100 - 3 /* space * / */ ) { + this->lfcr(); + } else { + this->writeSpace(); + } + this->writeCommentTrailer(); + } + lastEnd = token->fContentEnd; + this->lfcr(); + if (',' == fStart[0]) { + ++fStart; + } + this->writeBlock((int) (lastEnd - fStart), fStart); // enum item name + fStart = token->fContentEnd; + state = State::kItemName; + } +} + +void IncludeWriter::enumSizeItems(const Definition& child) { + enum class State { + kNoItem, + kItemName, + kItemValue, + kItemComment, + }; + State state = State::kNoItem; + int longestName = 0; + int longestValue = 0; + int valueLen = 0; + const char* lastEnd = nullptr; + SkASSERT(child.fChildren.size() == 1 || child.fChildren.size() == 2); + auto brace = child.fChildren[0]; + SkASSERT(Bracket::kBrace == brace->fBracket); + for (auto& token : brace->fTokens) { + if (Definition::Type::kBracket == token.fType) { + if (Bracket::kSlashSlash == token.fBracket) { + continue; // ignore old inline comments + } + if (Bracket::kSlashStar == token.fBracket) { + continue; // ignore old inline comments + } + SkASSERT(0); // incomplete + } + if (Definition::Type::kWord != token.fType) { + SkASSERT(0); // incomplete + } + if (State::kItemName == state) { + TextParser enumLine(token.fFileName, lastEnd, + token.fContentStart, token.fLineCount); + const char* end = enumLine.anyOf(",}="); + SkASSERT(end); + state = '=' == *end ? State::kItemValue : State::kItemComment; + if (State::kItemValue == state) { + valueLen = (int) (token.fContentEnd - token.fContentStart); + lastEnd = token.fContentEnd; + continue; + } + } + if (State::kItemValue == state) { + TextParser valueEnd(token.fFileName, lastEnd, + token.fContentStart, token.fLineCount); + const char* end = valueEnd.anyOf(",}"); + if (!end) { // write expression continuation + valueLen += (int) (token.fContentEnd - lastEnd); + continue; + } + } + if (State::kNoItem != state) { + longestValue = SkTMax(longestValue, valueLen); + state = State::kNoItem; + } + SkASSERT(State::kNoItem == state); + lastEnd = token.fContentEnd; + longestName = SkTMax(longestName, (int) (lastEnd - token.fContentStart)); + state = State::kItemName; + } + if (State::kItemValue == state) { + longestValue = SkTMax(longestValue, valueLen); + } + fEnumItemValueTab = longestName + fIndent + 1 /* space before = */ ; + if (longestValue) { + longestValue += 3; /* = space , */ + } + fEnumItemCommentTab = fEnumItemValueTab + longestValue + 1 /* space before //!< */ ; + // iterate through bmh children and see which comments fit on include lines + for (auto& enumItem : fEnumDef->fChildren) { + if (MarkType::kConst != enumItem->fMarkType) { + continue; + } + TextParser enumLine(enumItem); + enumLine.trimEnd(); + enumLine.skipToLineStart(); // skip const value + const char* commentStart = enumLine.fChar; + enumLine.skipLine(); + ptrdiff_t lineLen = enumLine.fChar - commentStart + 5 /* //!< space */ ; + if (!enumLine.eof()) { + enumLine.skipWhiteSpace(); + } + enumItem->fShort = enumLine.eof() && fEnumItemCommentTab + lineLen < 100; + } +} + +// walk children and output complete method doxygen description +void IncludeWriter::methodOut(const Definition* method) { + fContinuation = nullptr; + fDeferComment = nullptr; + if (0 == fIndent) { + fIndent = 4; + } + this->writeCommentHeader(); + fIndent += 4; + const char* commentStart = method->fContentStart; + int commentLen = (int) (method->fContentEnd - commentStart); + bool breakOut = false; + for (auto methodProp : method->fChildren) { + switch (methodProp->fMarkType) { + case MarkType::kDefinedBy: + commentStart = methodProp->fTerminator; + break; + case MarkType::kDeprecated: + case MarkType::kPrivate: + commentLen = (int) (methodProp->fStart - commentStart); + if (commentLen > 0) { + SkASSERT(commentLen < 1000); + if (Wrote::kNone != this->rewriteBlock(commentLen, commentStart)) { + this->lfcr(); + } + } + commentStart = methodProp->fContentStart; + commentLen = (int) (methodProp->fContentEnd - commentStart); + if (commentLen > 0) { + if (Wrote::kNone != this->rewriteBlock(commentLen, commentStart)) { + this->lfcr(); + } + } + commentStart = methodProp->fTerminator; + commentLen = (int) (method->fContentEnd - commentStart); + break; + default: + commentLen = (int) (methodProp->fStart - commentStart); + breakOut = true; + } + if (breakOut) { + break; + } + } + SkASSERT(commentLen > 0 && commentLen < 1000); + this->rewriteBlock(commentLen, commentStart); + // compute indention column + size_t column = 0; + bool hasParmReturn = false; + for (auto methodPart : method->fChildren) { + if (MarkType::kParam == methodPart->fMarkType) { + column = SkTMax(column, methodPart->fName.length()); + hasParmReturn = true; + } else if (MarkType::kReturn == methodPart->fMarkType) { + hasParmReturn = true; + } + } + if (hasParmReturn) { + this->lf(2); + column += fIndent + sizeof("@return "); + int saveIndent = fIndent; + for (auto methodPart : method->fChildren) { + const char* partStart = methodPart->fContentStart; + const char* partEnd = methodPart->fContentEnd; + if (MarkType::kParam == methodPart->fMarkType) { + this->writeString("@param"); + this->writeSpace(); + this->writeString(methodPart->fName.c_str()); + } else if (MarkType::kReturn == methodPart->fMarkType) { + this->writeString("@return"); + } else { + continue; + } + while ('\n' == partEnd[-1]) { + --partEnd; + } + while ('#' == partEnd[-1]) { // FIXME: so wrong; should not be before fContentEnd + --partEnd; + } + this->indentToColumn(column); + int partLen = (int) (partEnd - partStart); + SkASSERT(partLen > 0 && partLen < 200); + fIndent = column; + this->rewriteBlock(partLen, partStart); + fIndent = saveIndent; + this->lfcr(); + } + } else { + this->lfcr(); + } + fIndent -= 4; + this->lfcr(); + this->writeCommentTrailer(); +} + +void IncludeWriter::structOut(const Definition* root, const Definition& child, + const char* commentStart, const char* commentEnd) { + this->writeCommentHeader(); + this->writeString("\\"); + SkASSERT(MarkType::kClass == child.fMarkType || MarkType::kStruct == child.fMarkType); + this->writeString(MarkType::kClass == child.fMarkType ? "class" : "struct"); + this->writeSpace(); + this->writeString(child.fName.c_str()); + fIndent += 4; + this->lfcr(); + this->rewriteBlock((int) (commentEnd - commentStart), commentStart); + fIndent -= 4; + this->lfcr(); + this->writeCommentTrailer(); +} + +void IncludeWriter::structMemberOut(const Definition* memberStart, const Definition& child) { + const char* commentStart = nullptr; + ptrdiff_t commentLen = 0; + string name(child.fContentStart, (int) (child.fContentEnd - child.fContentStart)); + bool isShort; + for (auto memberDef : fStructDef->fChildren) { + if (memberDef->fName.length() - name.length() == memberDef->fName.find(name)) { + commentStart = memberDef->fContentStart; + commentLen = memberDef->fContentEnd - memberDef->fContentStart; + isShort = memberDef->fShort; + break; + } + } + if (!isShort) { + this->writeCommentHeader(); + fIndent += 4; + bool wroteLineFeed = Wrote::kLF == this->rewriteBlock(commentLen, commentStart); + fIndent -= 4; + if (wroteLineFeed || fColumn > 100 - 3 /* space * / */ ) { + this->lfcr(); + } else { + this->writeSpace(); + } + this->writeCommentTrailer(); + } + this->lfcr(); + this->writeBlock((int) (memberStart->fContentEnd - memberStart->fContentStart), + memberStart->fContentStart); + this->indentToColumn(fStructMemberTab); + this->writeString(name.c_str()); + this->writeString(";"); + if (isShort) { + this->indentToColumn(fStructCommentTab); + this->writeString("//!<"); + this->writeSpace(); + this->rewriteBlock(commentLen, commentStart); + this->lfcr(); + } +} + +void IncludeWriter::structSizeMembers(Definition& child) { + int longestType = 0; + Definition* typeStart = nullptr; + int longestName = 0; + SkASSERT(child.fChildren.size() == 1 || child.fChildren.size() == 2); + bool inEnum = false; + auto brace = child.fChildren[0]; + SkASSERT(Bracket::kBrace == brace->fBracket); + for (auto& token : brace->fTokens) { + if (Definition::Type::kBracket == token.fType) { + if (Bracket::kSlashSlash == token.fBracket) { + continue; // ignore old inline comments + } + if (Bracket::kSlashStar == token.fBracket) { + continue; // ignore old inline comments + } + if (Bracket::kParen == token.fBracket) { + break; + } + SkASSERT(0); // incomplete + } + if (Definition::Type::kKeyWord == token.fType) { + switch (token.fKeyWord) { + case KeyWord::kEnum: + inEnum = true; + break; + case KeyWord::kConst: + case KeyWord::kConstExpr: + case KeyWord::kStatic: + case KeyWord::kInt: + case KeyWord::kUint32_t: + case KeyWord::kSize_t: + case KeyWord::kFloat: + case KeyWord::kBool: + case KeyWord::kVoid: + if (!typeStart) { + typeStart = &token; + } + break; + default: + break; + } + continue; + } + if (Definition::Type::kPunctuation == token.fType) { + if (inEnum) { + SkASSERT(Punctuation::kSemicolon == token.fPunctuation); + inEnum = false; + } + continue; + } + if (Definition::Type::kWord != token.fType) { + SkASSERT(0); // incomplete + } + if (MarkType::kMember == token.fMarkType) { + TextParser typeStr(token.fFileName, typeStart->fContentStart, token.fContentStart, + token.fLineCount); + typeStr.trimEnd(); + longestType = SkTMax(longestType, (int) (typeStr.fEnd - typeStart->fContentStart)); + longestName = SkTMax(longestName, (int) (token.fContentEnd - token.fContentStart)); + typeStart->fMemberStart = true; + typeStart = nullptr; + continue; + } + SkASSERT(MarkType::kNone == token.fMarkType); + if (!typeStart) { + typeStart = &token; + } + } + fStructMemberTab = longestType + fIndent + 1 /* space before name */ ; + fStructCommentTab = fStructMemberTab + longestName + 2 /* ; space */ ; + // iterate through bmh children and see which comments fit on include lines + for (auto& member : fStructDef->fChildren) { + if (MarkType::kMember != member->fMarkType) { + continue; + } + TextParser memberLine(member); + memberLine.trimEnd(); + const char* commentStart = memberLine.fChar; + memberLine.skipLine(); + ptrdiff_t lineLen = memberLine.fChar - commentStart + 5 /* //!< space */ ; + if (!memberLine.eof()) { + memberLine.skipWhiteSpace(); + } + member->fShort = memberLine.eof() && fStructCommentTab + lineLen < 100; + } +} + +bool IncludeWriter::populate(Definition* def, RootDefinition* root) { + // write bulk of original include up to class, method, enum, etc., excepting preceding comment + // find associated bmh object + // write any associated comments in Doxygen form + // skip include comment + // if there is a series of same named methods, write one set of comments, then write all methods + string methodName; + const Definition* method; + const Definition* clonedMethod = nullptr; + const Definition* memberStart = nullptr; + fContinuation = nullptr; + bool inStruct = false; + for (auto& child : def->fTokens) { + if (child.fPrivate) { + continue; + } + if (fContinuation) { + if (Definition::Type::kKeyWord == child.fType) { + if (KeyWord::kFriend == child.fKeyWord || KeyWord::kBool == child.fKeyWord) { + continue; + } + } + if (Definition::Type::kBracket == child.fType && Bracket::kParen == child.fBracket) { + if (!clonedMethod) { + continue; + } + int alternate = 1; + ptrdiff_t childLen = child.fContentEnd - child.fContentStart; + SkASSERT(')' == child.fContentStart[childLen]); + ++childLen; + do { + TextParser params(clonedMethod->fFileName, clonedMethod->fStart, + clonedMethod->fContentStart, clonedMethod->fLineCount); + params.skipToEndBracket('('); + if (params.fEnd - params.fChar >= childLen && + !strncmp(params.fChar, child.fContentStart, childLen)) { + this->methodOut(clonedMethod); + break; + } + ++alternate; + string alternateMethod = methodName + '_' + to_string(alternate); + clonedMethod = root->find(alternateMethod); + } while (clonedMethod); + if (!clonedMethod) { + return this->reportError<bool>("cloned method not found"); + } + clonedMethod = nullptr; + continue; + } + if (Definition::Type::kWord == child.fType) { + if (clonedMethod) { + continue; + } + size_t len = (size_t) (child.fContentEnd - child.fContentStart); + const char operatorStr[] = "operator"; + size_t operatorLen = sizeof(operatorStr) - 1; + if (len >= operatorLen && !strncmp(child.fContentStart, operatorStr, operatorLen)) { + fContinuation = child.fContentEnd; + continue; + } + } + if (Definition::Type::kPunctuation == child.fType && + (Punctuation::kSemicolon == child.fPunctuation || + Punctuation::kLeftBrace == child.fPunctuation)) { + SkASSERT(fContinuation[0] == '('); + const char* continueEnd = child.fContentStart; + while (continueEnd > fContinuation && isspace(continueEnd[-1])) { + --continueEnd; + } + methodName += string(fContinuation, continueEnd - fContinuation); + method = root->find(methodName); + if (!method) { + fLineCount = child.fLineCount; + fclose(fOut); // so we can see what we've written so far + return this->reportError<bool>("method not found"); + } + this->methodOut(method); + continue; + } + methodName += "()"; + method = root->find(methodName); + if (MarkType::kDefinedBy == method->fMarkType) { + method = method->fParent; + } + if (method) { + this->methodOut(method); + continue; + } + fLineCount = child.fLineCount; + fclose(fOut); // so we can see what we've written so far + return this->reportError<bool>("method not found"); + } + if (Bracket::kSlashSlash == child.fBracket || Bracket::kSlashStar == child.fBracket) { + if (!fDeferComment) { + fDeferComment = &child; + } + continue; + } + if (MarkType::kMethod == child.fMarkType) { + const char* bodyEnd = fDeferComment ? fDeferComment->fContentStart - 1 : + child.fContentStart; + // FIXME: roll end-trimming into writeBlockTrim call + while (fStart < bodyEnd && ' ' >= bodyEnd[-1]) { + --bodyEnd; + } + int blockSize = (int) (bodyEnd - fStart); + if (blockSize) { + this->writeBlock(blockSize, fStart); + } + fStart = child.fContentStart; + methodName = root->fName + "::" + child.fName; + fContinuation = child.fContentEnd; + method = root->find(methodName); + if (!method) { + continue; + } + if (method->fCloned) { + clonedMethod = method; + continue; + } + this->methodOut(method); + continue; + } + if (Definition::Type::kKeyWord == child.fType) { + const Definition* structDef = nullptr; + switch (child.fKeyWord) { + case KeyWord::kStruct: + // if struct contains members, compute their name and comment tabs + inStruct = fInStruct = child.fChildren.size() > 0; + if (fInStruct) { + fIndent += 4; + fStructDef = root->find(child.fName); + if (nullptr == structDef) { + fStructDef = root->find(root->fName + "::" + child.fName); + } + this->structSizeMembers(child); + fIndent -= 4; + } + case KeyWord::kClass: + if (child.fChildren.size() > 0) { + const char* bodyEnd = fDeferComment ? fDeferComment->fContentStart - 1 : + child.fContentStart; + this->writeBlock((int) (bodyEnd - fStart), fStart); + fStart = child.fContentStart; + if (child.fName == root->fName) { + if (Definition* parent = root->fParent) { + if (MarkType::kTopic == parent->fMarkType || + MarkType::kSubtopic == parent->fMarkType) { + const char* commentStart = parent->fContentStart; + const char* commentEnd = root->fStart; + this->structOut(root, *root, commentStart, commentEnd); + } else { + SkASSERT(0); // incomplete + } + } else { + SkASSERT(0); // incomplete + } + } else { + structDef = root->find(child.fName); + if (nullptr == structDef) { + structDef = root->find(root->fName + "::" + child.fName); + } + Definition* codeBlock = nullptr; + Definition* nextBlock = nullptr; + for (auto test : structDef->fChildren) { + if (MarkType::kCode == test->fMarkType) { + SkASSERT(!codeBlock); // FIXME: check enum for correct order earlier + codeBlock = test; + continue; + } + if (codeBlock) { + nextBlock = test; + break; + } + } + SkASSERT(nextBlock); // FIXME: check enum for correct order earlier + const char* commentStart = codeBlock->fTerminator; + const char* commentEnd = nextBlock->fStart; + this->structOut(root, *structDef, commentStart, commentEnd); + } + fDeferComment = nullptr; + } else { + ; // empty forward reference, nothing to do here + } + break; + case KeyWord::kEnum: { + this->fInEnum = true; + this->enumHeaderOut(root, child); + this->enumSizeItems(child); + } break; + case KeyWord::kConst: + case KeyWord::kConstExpr: + case KeyWord::kStatic: + case KeyWord::kInt: + case KeyWord::kUint32_t: + case KeyWord::kSize_t: + case KeyWord::kFloat: + case KeyWord::kBool: + case KeyWord::kVoid: + if (!memberStart) { + memberStart = &child; + } + break; + case KeyWord::kPublic: + case KeyWord::kPrivate: + case KeyWord::kProtected: + case KeyWord::kFriend: + break; + default: + SkASSERT(0); + } + if (structDef) { + TextParser structName(&child); + SkAssertResult(structName.skipToEndBracket('{')); + fStart = structName.fChar + 1; + this->writeBlock((int) (fStart - child.fStart), child.fStart); + this->lf(2); + fIndent += 4; + if (!this->populate(&child, const_cast<Definition*>(structDef)->asRoot())) { + return false; + } + // output any remaining definitions at current indent level + const char* structEnd = child.fContentEnd; + SkAssertResult('}' == structEnd[-1]); + --structEnd; + this->writeBlock((int) (structEnd - fStart), fStart); + this->lf(2); + fStart = structEnd; + fIndent -= 4; + fContinuation = nullptr; + fDeferComment = nullptr; + } else { + if (!this->populate(&child, root)) { + return false; + } + } + continue; + } + if (Definition::Type::kBracket == child.fType) { + if (KeyWord::kEnum == child.fParent->fKeyWord) { + this->enumMembersOut(root, child); + this->writeString("};"); + this->lf(2); + fStart = child.fParent->fContentEnd; + SkASSERT(';' == fStart[0]); + ++fStart; + fDeferComment = nullptr; + fInEnum = false; + continue; + } + fDeferComment = nullptr; + if (!this->populate(&child, root)) { + return false; + } + continue; + } + if (Definition::Type::kWord == child.fType) { + if (MarkType::kMember == child.fMarkType) { + this->structMemberOut(memberStart, child); + fStart = child.fContentEnd + 1; + fDeferComment = nullptr; + } + if (child.fMemberStart) { + memberStart = &child; + } + continue; + } + if (Definition::Type::kPunctuation == child.fType) { + if (Punctuation::kSemicolon == child.fPunctuation) { + memberStart = nullptr; + if (inStruct) { + fInStruct = false; + } + continue; + } + if (Punctuation::kLeftBrace == child.fPunctuation || + Punctuation::kColon == child.fPunctuation || + Punctuation::kAsterisk == child.fPunctuation + ) { + continue; + } + } + } + return true; +} + +bool IncludeWriter::populate(BmhParser& bmhParser) { + bool allPassed = true; + for (auto& includeMapper : fIncludeMap) { + size_t lastSlash = includeMapper.first.rfind('/'); + if (string::npos == lastSlash || lastSlash >= includeMapper.first.length() - 1) { + return this->reportError<bool>("malformed include name"); + } + string fileName = includeMapper.first.substr(lastSlash + 1); + if (".h" != fileName.substr(fileName.length() - 2)) { + return this->reportError<bool>("expected fileName.h"); + } + string skClassName = fileName.substr(0, fileName.length() - 2); + fOut = fopen(fileName.c_str(), "wb"); + if (!fOut) { + SkDebugf("could not open output file %s\n", fileName.c_str()); + return false; + } + if (bmhParser.fClassMap.end() == bmhParser.fClassMap.find(skClassName)) { + return this->reportError<bool>("could not find bmh class"); + } + fBmhParser = &bmhParser; + RootDefinition* root = &bmhParser.fClassMap[skClassName]; + fRootTopic = root->fParent; + root->clearVisited(); + fStart = includeMapper.second.fContentStart; + fEnd = includeMapper.second.fContentEnd; + allPassed &= this->populate(&includeMapper.second, root); + this->writeBlock((int) (fEnd - fStart), fStart); + fIndent = 0; + this->lfcr(); + this->writePending(); + fclose(fOut); + } + return allPassed; +} + +// change Xxx_Xxx to xxx xxx +static string ConvertRef(const string str, bool first) { + string substitute; + for (char c : str) { + if ('_' == c) { + c = ' '; // change Xxx_Xxx to xxx xxx + } else if (isupper(c) && !first) { + c = tolower(c); + } + substitute += c; + first = false; + } + return substitute; +} + +// FIXME: buggy that this is called with strings containing spaces but resolveRef below is not.. +string IncludeWriter::resolveMethod(const char* start, const char* end, bool first) { + string methodname(start, end - start); + string substitute; + auto rootDefIter = fBmhParser->fMethodMap.find(methodname); + if (fBmhParser->fMethodMap.end() != rootDefIter) { + substitute = methodname + "()"; + } else { + auto parent = fRootTopic->fChildren[0]->asRoot(); + auto defRef = parent->find(parent->fName + "::" + methodname); + if (defRef && MarkType::kMethod == defRef->fMarkType) { + substitute = methodname + "()"; + } + } + return substitute; +} + +string IncludeWriter::resolveRef(const char* start, const char* end, bool first) { + // look up Xxx_Xxx + string undername(start, end - start); + SkASSERT(string::npos == undername.find(' ')); + const Definition* rootDef = nullptr; + { + auto rootDefIter = fBmhParser->fTopicMap.find(undername); + if (fBmhParser->fTopicMap.end() != rootDefIter) { + rootDef = rootDefIter->second; + } else { + string prefixedName = fRootTopic->fName + '_' + undername; + rootDefIter = fBmhParser->fTopicMap.find(prefixedName); + if (fBmhParser->fTopicMap.end() != rootDefIter) { + rootDef = rootDefIter->second; + } else { + auto aliasIter = fBmhParser->fAliasMap.find(undername); + if (fBmhParser->fAliasMap.end() != aliasIter) { + rootDef = aliasIter->second->fParent; + } else if (!first) { + for (const auto& external : fBmhParser->fExternals) { + if (external.fName == undername) { + return external.fName; + } + } + SkDebugf("unfound: %s\n", undername.c_str()); + } + } + } + } + string substitute; + if (rootDef) { + for (auto child : rootDef->fChildren) { + if (MarkType::kSubstitute == child->fMarkType) { + substitute = string(child->fContentStart, + (int) (child->fContentEnd - child->fContentStart)); + break; + } + if (MarkType::kClass == child->fMarkType || + MarkType::kStruct == child->fMarkType || + MarkType::kEnum == child->fMarkType || + MarkType::kEnumClass == child->fMarkType) { + substitute = child->fName; + if (MarkType::kEnum == child->fMarkType && fInEnum) { + size_t parentClassEnd = substitute.find("::"); + SkASSERT(string::npos != parentClassEnd); + substitute = substitute.substr(parentClassEnd + 2); + } + break; + } + } + if (!substitute.length()) { + auto parent = rootDef->fParent; + if (parent) { + if (MarkType::kClass == parent->fMarkType || + MarkType::kStruct == parent->fMarkType || + MarkType::kEnum == parent->fMarkType || + MarkType::kEnumClass == parent->fMarkType) { + if (parent->fParent != fRootTopic) { + substitute = parent->fName; + size_t under = undername.find('_'); + SkASSERT(string::npos != under); + string secondHalf(&undername[under], (size_t) (undername.length() - under)); + substitute += ConvertRef(secondHalf, false); + } else { + substitute += ConvertRef(undername, first); + } + } + } + } + } + // start here; + // first I thought first meant first word after period, but the below doesn't work +// if (first && isupper(start[0]) && substitute.length() > 0 && islower(substitute[0])) { +// substitute[0] = start[0]; +// } + return substitute; +} +int IncludeWriter::lookupMethod(const PunctuationState punctuation, const Word word, + const int start, const int run, int lastWrite, const char last, const char* data) { + const int end = PunctuationState::kDelimiter == punctuation || + PunctuationState::kPeriod == punctuation ? run - 1 : run; + string temp = this->resolveMethod(&data[start], &data[end], Word::kFirst == word); + if (temp.length()) { + if (start > lastWrite) { + SkASSERT(data[start - 1] >= ' '); + if (' ' == data[lastWrite]) { + this->writeSpace(); + } + this->writeBlockTrim(start - lastWrite, &data[lastWrite]); + if (' ' == data[start - 1]) { + this->writeSpace(); + } + } + SkASSERT(temp[temp.length() - 1] > ' '); + this->writeString(temp.c_str()); + lastWrite = end; + } + return lastWrite; +} + +int IncludeWriter::lookupReference(const PunctuationState punctuation, const Word word, + const int start, const int run, int lastWrite, const char last, const char* data) { + const int end = PunctuationState::kDelimiter == punctuation || + PunctuationState::kPeriod == punctuation ? run - 1 : run; + string temp = this->resolveRef(&data[start], &data[end], Word::kFirst == word); + if (!temp.length()) { + if (Word::kFirst != word && '_' != last) { + temp = string(&data[start], (size_t) (end - start)); + temp = ConvertRef(temp, false); + } + } + if (temp.length()) { + if (start > lastWrite) { + SkASSERT(data[start - 1] >= ' '); + if (' ' == data[lastWrite]) { + this->writeSpace(); + } + this->writeBlockTrim(start - lastWrite, &data[lastWrite]); + if (' ' == data[start - 1]) { + this->writeSpace(); + } + } + SkASSERT(temp[temp.length() - 1] > ' '); + this->writeString(temp.c_str()); + lastWrite = end; + } + return lastWrite; +} + +/* returns true if rewriteBlock wrote linefeeds */ +IncludeWriter::Wrote IncludeWriter::rewriteBlock(int size, const char* data) { + bool wroteLineFeeds = false; + while (size > 0 && data[0] <= ' ') { + --size; + ++data; + } + while (size > 0 && data[size - 1] <= ' ') { + --size; + } + if (0 == size) { + return Wrote::kNone; + } + int run = 0; + Word word = Word::kStart; + PunctuationState punctuation = PunctuationState::kStart; + int start = 0; + int lastWrite = 0; + int lineFeeds = 0; + int lastPrintable = 0; + char c = 0; + char last; + bool hasLower = false; + bool hasUpper = false; + bool hasSymbol = false; + while (run < size) { + last = c; + c = data[run]; + SkASSERT(' ' <= c || '\n' == c); + if (lineFeeds && ' ' < c) { + if (lastPrintable >= lastWrite) { + if (' ' == data[lastWrite]) { + this->writeSpace(); + } + this->writeBlock(lastPrintable - lastWrite + 1, &data[lastWrite]); + } + if (lineFeeds > 1) { + this->lf(2); + } + this->lfcr(); // defer the indent until non-whitespace is seen + lastWrite = run; + lineFeeds = 0; + } + if (' ' < c) { + lastPrintable = run; + } + switch (c) { + case '\n': + ++lineFeeds; + wroteLineFeeds = true; + case ' ': + switch (word) { + case Word::kStart: + break; + case Word::kUnderline: + case Word::kCap: + case Word::kFirst: + if (!hasLower) { + break; + } + lastWrite = this->lookupReference(punctuation, word, start, run, + lastWrite, last, data); + break; + case Word::kMixed: + if (hasUpper && hasLower && !hasSymbol) { + lastWrite = this->lookupMethod(punctuation, word, start, run, + lastWrite, last, data); + } + break; + default: + SkASSERT(0); + } + punctuation = PunctuationState::kStart; + word = Word::kStart; + hasLower = false; + hasUpper = false; + hasSymbol = false; + break; + case '.': + switch (word) { + case Word::kStart: + punctuation = PunctuationState::kDelimiter; + case Word::kCap: + case Word::kFirst: + case Word::kUnderline: + case Word::kMixed: + if (PunctuationState::kDelimiter == punctuation || + PunctuationState::kPeriod == punctuation) { + word = Word::kMixed; + } + punctuation = PunctuationState::kPeriod; + break; + default: + SkASSERT(0); + } + hasSymbol = true; + break; + case ',': case ';': case ':': + hasSymbol |= PunctuationState::kDelimiter == punctuation; + switch (word) { + case Word::kStart: + punctuation = PunctuationState::kDelimiter; + case Word::kCap: + case Word::kFirst: + case Word::kUnderline: + case Word::kMixed: + if (PunctuationState::kDelimiter == punctuation || + PunctuationState::kPeriod == punctuation) { + word = Word::kMixed; + } + punctuation = PunctuationState::kDelimiter; + break; + default: + SkASSERT(0); + } + break; + case '\'': // possessive apostrophe isn't treated as delimiting punctation + case '=': + case '!': // assumed not to be punctuation, but a programming symbol + case '&': case '>': case '<': case '{': case '}': case '/': case '*': + word = Word::kMixed; + hasSymbol = true; + break; + case '(': + if (' ' == last) { + punctuation = PunctuationState::kDelimiter; + } else { + word = Word::kMixed; + } + hasSymbol = true; + break; + case ')': // assume word type has already been set + punctuation = PunctuationState::kDelimiter; + hasSymbol = true; + break; + case '_': + switch (word) { + case Word::kStart: + word = Word::kMixed; + break; + case Word::kCap: + case Word::kFirst: + case Word::kUnderline: + word = Word::kUnderline; + break; + case Word::kMixed: + break; + default: + SkASSERT(0); + } + break; + case 'A': case 'B': case 'C': case 'D': case 'E': + case 'F': case 'G': case 'H': case 'I': case 'J': + case 'K': case 'L': case 'M': case 'N': case 'O': + case 'P': case 'Q': case 'R': case 'S': case 'T': + case 'U': case 'V': case 'W': case 'X': case 'Y': + case 'Z': + switch (word) { + case Word::kStart: + word = PunctuationState::kStart == punctuation ? Word::kFirst : Word::kCap; + start = run; + break; + case Word::kCap: + case Word::kFirst: + if (!isupper(last)) { + word = Word::kMixed; + } + break; + case Word::kUnderline: + // some word in Xxx_XXX_Xxx can be all upper, but all can't: XXX_XXX + if ('_' != last && !isupper(last)) { + word = Word::kMixed; + } + break; + case Word::kMixed: + break; + default: + SkASSERT(0); + } + hasUpper = true; + if (PunctuationState::kPeriod == punctuation || + PunctuationState::kDelimiter == punctuation) { + word = Word::kMixed; + } else { + punctuation = PunctuationState::kStart; + } + break; + case 'a': case 'b': case 'c': case 'd': case 'e': + case 'f': case 'g': case 'h': case 'i': case 'j': + case 'k': case 'l': case 'm': case 'n': case 'o': + case 'p': case 'q': case 'r': case 's': case 't': + case 'u': case 'v': case 'w': case 'x': case 'y': + case 'z': + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + case '-': + switch (word) { + case Word::kStart: + word = Word::kMixed; + break; + case Word::kMixed: + case Word::kCap: + case Word::kFirst: + case Word::kUnderline: + break; + default: + SkASSERT(0); + } + hasLower = true; + punctuation = PunctuationState::kStart; + break; + default: + SkASSERT(0); + } + ++run; + } + if ((word == Word::kCap || word == Word::kFirst || word == Word::kUnderline) && hasLower) { + lastWrite = this->lookupReference(punctuation, word, start, run, lastWrite, last, data); + } + if (run > lastWrite) { + if (' ' == data[lastWrite]) { + this->writeSpace(); + } + this->writeBlock(run - lastWrite, &data[lastWrite]); + } + return wroteLineFeeds ? Wrote::kLF : Wrote::kChars; +} diff --git a/tools/bookmaker/mdOut.cpp b/tools/bookmaker/mdOut.cpp new file mode 100644 index 0000000000..a7737d6d70 --- /dev/null +++ b/tools/bookmaker/mdOut.cpp @@ -0,0 +1,929 @@ +/* + * Copyright 2017 Google Inc. + * + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include "bookmaker.h" + +#include "SkOSFile.h" +#include "SkOSPath.h" + +static void add_ref(const string& leadingSpaces, const string& ref, string* result) { + *result += leadingSpaces + ref; +} + +// FIXME: preserve inter-line spaces and don't add new ones +string MdOut::addReferences(const char* refStart, const char* refEnd, + BmhParser::Resolvable resolvable) { + string result; + MethodParser t(fRoot ? fRoot->fName : string(), fFileName, refStart, refEnd, fLineCount); + bool lineStart = true; + string ref; + string leadingSpaces; + do { + const char* base = t.fChar; + t.skipWhiteSpace(); + const char* wordStart = t.fChar; + t.skipToMethodStart(); + const char* start = t.fChar; + if (wordStart < start) { + if (lineStart) { + lineStart = false; + } else { + wordStart = base; + } + result += string(wordStart, start - wordStart); + if ('\n' != result.back()) { + while (start > wordStart && '\n' == start[-1]) { + result += '\n'; + --start; + } + } + } + if (lineStart) { + lineStart = false; + } else { + leadingSpaces = string(base, wordStart - base); + } + t.skipToMethodEnd(); + if (base == t.fChar) { + break; + } + if (start >= t.fChar) { + continue; + } + if (!t.eof() && '"' == t.peek() && start > wordStart && '"' == start[-1]) { + continue; + } + ref = string(start, t.fChar - start); + if (const Definition* def = this->isDefined(t, ref, + BmhParser::Resolvable::kOut != resolvable)) { + SkASSERT(def->fFiddle.length()); + if (!t.eof() && '(' == t.peek() && t.strnchr(')', t.fEnd)) { + if (!t.skipToEndBracket(')')) { + t.reportError("missing close paren"); + return result; + } + t.next(); + string fullRef = string(start, t.fChar - start); + // if _2 etc alternates are defined, look for paren match + // may ignore () if ref is all lower case + // otherwise flag as error + int suffix = '2'; + bool foundMatch = false; + const Definition* altDef = def; + while (altDef && suffix <= '9') { + if ((foundMatch = altDef->paramsMatch(fullRef, ref))) { + def = altDef; + ref = fullRef; + break; + } + string altTest = ref + '_'; + altTest += suffix++; + altDef = this->isDefined(t, altTest, false); + } + if (suffix > '9') { + t.reportError("too many alts"); + return result; + } + if (!foundMatch) { + if (!(def = this->isDefined(t, fullRef, true))) { + return result; + } + ref = fullRef; + } + } + result += linkRef(leadingSpaces, def, ref); + continue; + } + if (!t.eof() && '(' == t.peek()) { + if (!t.skipToEndBracket(')')) { + t.reportError("missing close paren"); + return result; + } + t.next(); + ref = string(start, t.fChar - start); + if (const Definition* def = this->isDefined(t, ref, true)) { + SkASSERT(def->fFiddle.length()); + result += linkRef(leadingSpaces, def, ref); + continue; + } + } +// class, struct, and enum start with capitals +// methods may start with upper (static) or lower (most) + + // see if this should have been a findable reference + + // look for Sk / sk / SK .. + if (!ref.compare(0, 2, "Sk") && ref != "Skew" && ref != "Skews" && + ref != "Skip" && ref != "Skips") { + t.reportError("missed Sk prefixed"); + return result; + } + if (!ref.compare(0, 2, "SK")) { + if (BmhParser::Resolvable::kOut != resolvable) { + t.reportError("missed SK prefixed"); + } + return result; + } + if (!isupper(start[0])) { + // TODO: + // look for all lowercase w/o trailing parens as mistaken method matches + // will also need to see if Example Description matches var in example + const Definition* def; + if (fMethod && (def = fMethod->hasParam(ref))) { + result += linkRef(leadingSpaces, def, ref); + continue; + } else if (!fInDescription && ref[0] != '0' + && string::npos != ref.find_first_of("ABCDEFGHIJKLMNOPQRSTUVWXYZ")) { + // FIXME: see isDefined(); check to see if fXX is a member of xx.fXX + if (('f' != ref[0] && string::npos == ref.find("()")) + || '.' != t.backup(ref.c_str())) { + if (BmhParser::Resolvable::kOut != resolvable) { + t.reportError("missed camelCase"); + return result; + } + } + } + add_ref(leadingSpaces, ref, &result); + continue; + } + auto topicIter = fBmhParser.fTopicMap.find(ref); + if (topicIter != fBmhParser.fTopicMap.end()) { + result += linkRef(leadingSpaces, topicIter->second, ref); + continue; + } + bool startsSentence = t.sentenceEnd(start); + if (!t.eof() && ' ' != t.peek()) { + add_ref(leadingSpaces, ref, &result); + continue; + } + if (t.fChar + 1 >= t.fEnd || (!isupper(t.fChar[1]) && startsSentence)) { + add_ref(leadingSpaces, ref, &result); + continue; + } + if (isupper(t.fChar[1]) && startsSentence) { + TextParser next(t.fFileName, &t.fChar[1], t.fEnd, t.fLineCount); + string nextWord(next.fChar, next.wordEnd() - next.fChar); + if (this->isDefined(t, nextWord, true)) { + add_ref(leadingSpaces, ref, &result); + continue; + } + } + Definition* test = fRoot; + do { + if (!test->isRoot()) { + continue; + } + for (string prefix : { "_", "::" } ) { + RootDefinition* root = test->asRoot(); + string prefixed = root->fName + prefix + ref; + if (const Definition* def = root->find(prefixed)) { + result += linkRef(leadingSpaces, def, ref); + goto found; + } + } + } while ((test = test->fParent)); + found: + if (!test) { + if (BmhParser::Resolvable::kOut != resolvable) { + t.reportError("undefined reference"); + } + } + } while (!t.eof()); + return result; +} + +bool MdOut::buildReferences(const char* fileOrPath, const char* outDir) { + if (!sk_isdir(fileOrPath)) { + if (!this->buildRefFromFile(fileOrPath, outDir)) { + SkDebugf("failed to parse %s\n", fileOrPath); + return false; + } + } else { + SkOSFile::Iter it(fileOrPath, ".bmh"); + for (SkString file; it.next(&file); ) { + SkString p = SkOSPath::Join(fileOrPath, file.c_str()); + const char* hunk = p.c_str(); + if (!SkStrEndsWith(hunk, ".bmh")) { + continue; + } + if (SkStrEndsWith(hunk, "markup.bmh")) { // don't look inside this for now + continue; + } + if (!this->buildRefFromFile(hunk, outDir)) { + SkDebugf("failed to parse %s\n", hunk); + return false; + } + } + } + return true; +} + +bool MdOut::buildRefFromFile(const char* name, const char* outDir) { + fFileName = string(name); + string filename(name); + if (filename.substr(filename.length() - 4) == ".bmh") { + filename = filename.substr(0, filename.length() - 4); + } + size_t start = filename.length(); + while (start > 0 && (isalnum(filename[start - 1]) || '_' == filename[start - 1])) { + --start; + } + string match = filename.substr(start); + string header = match; + filename = "bmh_" + match + ".md"; + match += ".bmh"; + fOut = nullptr; + for (const auto& topic : fBmhParser.fTopicMap) { + Definition* topicDef = topic.second; + if (topicDef->fParent) { + continue; + } + if (!topicDef->isRoot()) { + return this->reportError<bool>("expected root topic"); + } + fRoot = topicDef->asRoot(); + if (string::npos == fRoot->fFileName.rfind(match)) { + continue; + } + if (!fOut) { + string fullName(outDir); + if ('/' != fullName.back()) { + fullName += '/'; + } + fullName += filename; + fOut = fopen(fullName.c_str(), "wb"); + if (!fOut) { + SkDebugf("could not open output file %s\n", fullName.c_str()); + return false; + } + fprintf(fOut, "Experimental %s", header.c_str()); + this->lfAlways(1); + fprintf(fOut, "==="); + } + this->markTypeOut(topicDef); + } + if (fOut) { + this->writePending(); + fclose(fOut); + fOut = nullptr; + } + return true; +} + +void MdOut::childrenOut(const Definition* def, const char* start) { + const char* end; + fLineCount = def->fLineCount; + if (def->isRoot()) { + fRoot = const_cast<RootDefinition*>(def->asRoot()); + } + BmhParser::Resolvable resolvable = this->resolvable(def->fMarkType); + for (auto& child : def->fChildren) { + end = child->fStart; + if (BmhParser::Resolvable::kNo != resolvable) { + this->resolveOut(start, end, resolvable); + } + this->markTypeOut(child); + start = child->fTerminator; + } + if (BmhParser::Resolvable::kNo != resolvable) { + end = def->fContentEnd; + this->resolveOut(start, end, resolvable); + } +} + +const Definition* MdOut::isDefined(const TextParser& parser, const string& ref, bool report) const { + auto rootIter = fBmhParser.fClassMap.find(ref); + if (rootIter != fBmhParser.fClassMap.end()) { + return &rootIter->second; + } + auto typedefIter = fBmhParser.fTypedefMap.find(ref); + if (typedefIter != fBmhParser.fTypedefMap.end()) { + return &typedefIter->second; + } + auto enumIter = fBmhParser.fEnumMap.find(ref); + if (enumIter != fBmhParser.fEnumMap.end()) { + return &enumIter->second; + } + auto constIter = fBmhParser.fConstMap.find(ref); + if (constIter != fBmhParser.fConstMap.end()) { + return &constIter->second; + } + auto methodIter = fBmhParser.fMethodMap.find(ref); + if (methodIter != fBmhParser.fMethodMap.end()) { + return &methodIter->second; + } + auto aliasIter = fBmhParser.fAliasMap.find(ref); + if (aliasIter != fBmhParser.fAliasMap.end()) { + return aliasIter->second; + } + for (const auto& external : fBmhParser.fExternals) { + if (external.fName == ref) { + return &external; + } + } + if (fRoot) { + if (ref == fRoot->fName) { + return fRoot; + } + if (const Definition* definition = fRoot->find(ref)) { + return definition; + } + Definition* test = fRoot; + do { + if (!test->isRoot()) { + continue; + } + RootDefinition* root = test->asRoot(); + for (auto& leaf : root->fBranches) { + if (ref == leaf.first) { + return leaf.second; + } + const Definition* definition = leaf.second->find(ref); + if (definition) { + return definition; + } + } + for (string prefix : { "::", "_" } ) { + string prefixed = root->fName + prefix + ref; + if (const Definition* definition = root->find(prefixed)) { + return definition; + } + if (isupper(prefixed[0])) { + auto topicIter = fBmhParser.fTopicMap.find(prefixed); + if (topicIter != fBmhParser.fTopicMap.end()) { + return topicIter->second; + } + } + } + } while ((test = test->fParent)); + } + size_t doubleColon = ref.find("::"); + if (string::npos != doubleColon) { + string className = ref.substr(0, doubleColon); + auto classIter = fBmhParser.fClassMap.find(className); + if (classIter != fBmhParser.fClassMap.end()) { + const RootDefinition& classDef = classIter->second; + const Definition* result = classDef.find(ref); + if (result) { + return result; + } + } + + } + if (!ref.compare(0, 2, "SK") || !ref.compare(0, 3, "sk_") + || (('k' == ref[0] || 'g' == ref[0] || 'f' == ref[0]) && + ref.length() > 1 && isupper(ref[1]))) { + // try with a prefix + if ('k' == ref[0]) { + for (auto const& iter : fBmhParser.fEnumMap) { + if (iter.second.find(ref)) { + return &iter.second; + } + } + } + if ('f' == ref[0]) { + // FIXME : find def associated with prior, e.g.: r.fX where 'SkPoint r' was earlier + // need to have pushed last resolve on stack to do this + // for now, just try to make sure that it's there and error if not + if ('.' != parser.backup(ref.c_str())) { + parser.reportError("fX member undefined"); + return nullptr; + } + } else { + if (report) { + parser.reportError("SK undefined"); + } + return nullptr; + } + } + if (isupper(ref[0])) { + auto topicIter = fBmhParser.fTopicMap.find(ref); + if (topicIter != fBmhParser.fTopicMap.end()) { + return topicIter->second; + } + size_t pos = ref.find('_'); + if (string::npos != pos) { + // see if it is defined by another base class + string className(ref, 0, pos); + auto classIter = fBmhParser.fClassMap.find(className); + if (classIter != fBmhParser.fClassMap.end()) { + if (const Definition* definition = classIter->second.find(ref)) { + return definition; + } + } + auto enumIter = fBmhParser.fEnumMap.find(className); + if (enumIter != fBmhParser.fEnumMap.end()) { + if (const Definition* definition = enumIter->second.find(ref)) { + return definition; + } + } + if (report) { + parser.reportError("_ undefined"); + } + return nullptr; + } + } + return nullptr; +} + +string MdOut::linkName(const Definition* ref) const { + string result = ref->fName; + size_t under = result.find('_'); + if (string::npos != under) { + string classPart = result.substr(0, under); + string namePart = result.substr(under + 1, result.length()); + if (fRoot && (fRoot->fName == classPart + || (fRoot->fParent && fRoot->fParent->fName == classPart))) { + result = namePart; + } + } + return result; +} + +// for now, hard-code to html links +// def should not include SkXXX_ +string MdOut::linkRef(const string& leadingSpaces, const Definition* def, + const string& ref) const { + string buildup; + const string* str = &def->fFiddle; + SkASSERT(str->length() > 0); + size_t under = str->find('_'); + Definition* curRoot = fRoot; + string classPart = string::npos != under ? str->substr(0, under) : *str; + bool classMatch = curRoot->fName == classPart; + while (curRoot->fParent) { + curRoot = curRoot->fParent; + classMatch |= curRoot->fName == classPart; + } + const Definition* defRoot; + do { + defRoot = def; + if (!(def = def->fParent)) { + break; + } + classMatch |= def != defRoot && def->fName == classPart; + } while (true); + string namePart = string::npos != under ? str->substr(under + 1, str->length()) : *str; + SkASSERT(fRoot); + SkASSERT(fRoot->fFileName.length()); + if (false && classMatch) { + str = &namePart; + } else if (true || (curRoot != defRoot && defRoot->isRoot())) { + string filename = defRoot->asRoot()->fFileName; + if (filename.substr(filename.length() - 4) == ".bmh") { + filename = filename.substr(0, filename.length() - 4); + } + size_t start = filename.length(); + while (start > 0 && (isalnum(filename[start - 1]) || '_' == filename[start - 1])) { + --start; + } + buildup = "bmh_" + filename.substr(start) + "?cl=9919#" + + (classMatch ? namePart : *str); + str = &buildup; + } + string refOut(ref); + std::replace(refOut.begin(), refOut.end(), '_', ' '); + if (ref.length() > 2 && islower(ref[0]) && "()" == ref.substr(ref.length() - 2)) { + refOut = refOut.substr(0, refOut.length() - 2); + } + return leadingSpaces + "<a href=\"" + *str + "\">" + refOut + "</a>"; +} + +void MdOut::markTypeOut(Definition* def) { + string printable = def->printableName(); + const char* textStart = def->fContentStart; + if (MarkType::kParam != def->fMarkType && MarkType::kConst != def->fMarkType && + (!def->fParent || MarkType::kConst != def->fParent->fMarkType) && + TableState::kNone != fTableState) { + this->writePending(); + fprintf(fOut, "</table>"); + this->lf(2); + fTableState = TableState::kNone; + } + switch (def->fMarkType) { + case MarkType::kAlias: + break; + case MarkType::kAnchor: + break; + case MarkType::kBug: + break; + case MarkType::kClass: + this->mdHeaderOut(1); + fprintf(fOut, "<a name=\"%s\"></a> Class %s", this->linkName(def).c_str(), + def->fName.c_str()); + this->lf(1); + break; + case MarkType::kCode: + this->lfAlways(2); + fprintf(fOut, "<pre style=\"padding: 1em 1em 1em 1em;" + "width: 44em; background-color: #f0f0f0\">"); + this->lf(1); + break; + case MarkType::kColumn: + this->writePending(); + if (fInList) { + fprintf(fOut, " <td>"); + } else { + fprintf(fOut, "| "); + } + break; + case MarkType::kComment: + break; + case MarkType::kConst: { + if (TableState::kNone == fTableState) { + this->mdHeaderOut(3); + fprintf(fOut, "Constants\n" + "\n" + "<table>"); + fTableState = TableState::kRow; + this->lf(1); + } + if (TableState::kRow == fTableState) { + this->writePending(); + fprintf(fOut, " <tr>"); + this->lf(1); + fTableState = TableState::kColumn; + } + this->writePending(); + fprintf(fOut, " <td><a name=\"%s\"></a> <code><strong>%s </strong></code></td>", + def->fName.c_str(), def->fName.c_str()); + const char* lineEnd = strchr(textStart, '\n'); + SkASSERT(lineEnd < def->fTerminator); + SkASSERT(lineEnd > textStart); + SkASSERT((int) (lineEnd - textStart) == lineEnd - textStart); + fprintf(fOut, "<td>%.*s</td>", (int) (lineEnd - textStart), textStart); + fprintf(fOut, "<td>"); + textStart = lineEnd; + } break; + case MarkType::kDefine: + break; + case MarkType::kDefinedBy: + break; + case MarkType::kDeprecated: + break; + case MarkType::kDescription: + fInDescription = true; + this->writePending(); + fprintf(fOut, "<div>"); + break; + case MarkType::kDoxygen: + break; + case MarkType::kEnum: + case MarkType::kEnumClass: + this->mdHeaderOut(2); + fprintf(fOut, "<a name=\"%s\"></a> Enum %s", def->fName.c_str(), def->fName.c_str()); + this->lf(2); + break; + case MarkType::kError: + break; + case MarkType::kExample: { + this->mdHeaderOut(3); + fprintf(fOut, "Example\n" + "\n"); + fHasFiddle = true; + const Definition* platform = def->hasChild(MarkType::kPlatform); + if (platform) { + TextParser platParse(platform); + fHasFiddle = !platParse.strnstr("!fiddle", platParse.fEnd); + } + if (fHasFiddle) { + fprintf(fOut, "<div><fiddle-embed name=\"%s\">", def->fHash.c_str()); + } else { + fprintf(fOut, "<pre style=\"padding: 1em 1em 1em 1em;" + "width: 44em; background-color: #f0f0f0\">"); + this->lf(1); + } + } break; + case MarkType::kExperimental: + break; + case MarkType::kExternal: + break; + case MarkType::kFile: + break; + case MarkType::kFormula: + break; + case MarkType::kFunction: + break; + case MarkType::kHeight: + break; + case MarkType::kImage: + break; + case MarkType::kLegend: + break; + case MarkType::kLink: + break; + case MarkType::kList: + fInList = true; + this->lfAlways(2); + fprintf(fOut, "<table>"); + this->lf(1); + break; + case MarkType::kMarkChar: + fBmhParser.fMC = def->fContentStart[0]; + break; + case MarkType::kMember: { + TextParser tp(def->fFileName, def->fStart, def->fContentStart, def->fLineCount); + tp.skipExact("#Member"); + tp.skipWhiteSpace(); + const char* end = tp.trimmedBracketEnd('\n', TextParser::OneLine::kYes); + this->lfAlways(2); + fprintf(fOut, "<code><strong>%.*s</strong></code>", (int) (end - tp.fChar), tp.fChar); + this->lf(2); + } break; + case MarkType::kMethod: { + string method_name = def->methodName(); + string formattedStr = def->formatFunction(); + + if (!def->isClone()) { + this->lfAlways(2); + fprintf(fOut, "<a name=\"%s\"></a>", def->fiddleName().c_str()); + this->mdHeaderOutLF(2, 1); + fprintf(fOut, "%s", method_name.c_str()); + this->lf(2); + } + + // TODO: put in css spec that we can define somewhere else (if markup supports that) + // TODO: 50em below should match limt = 80 in formatFunction() + this->writePending(); + fprintf(fOut, "<pre style=\"padding: 1em 1em 1em 1em;" + "width: 50em; background-color: #f0f0f0\">\n" + "%s\n" + "</pre>", formattedStr.c_str()); + this->lf(2); + fTableState = TableState::kNone; + fMethod = def; + } break; + case MarkType::kNoExample: + break; + case MarkType::kParam: { + if (TableState::kNone == fTableState) { + this->mdHeaderOut(3); + fprintf(fOut, + "Parameters\n" + "\n" + "<table>" + ); + this->lf(1); + fTableState = TableState::kRow; + } + if (TableState::kRow == fTableState) { + fprintf(fOut, " <tr>"); + this->lf(1); + fTableState = TableState::kColumn; + } + TextParser paramParser(def->fFileName, def->fStart, def->fContentStart, + def->fLineCount); + paramParser.skipWhiteSpace(); + SkASSERT(paramParser.startsWith("#Param")); + paramParser.next(); // skip hash + paramParser.skipToNonAlphaNum(); // skip Param + paramParser.skipSpace(); + const char* paramName = paramParser.fChar; + paramParser.skipToSpace(); + fprintf(fOut, + " <td><code><strong>%.*s </strong></code></td> <td>", + (int) (paramParser.fChar - paramName), paramName); + } break; + case MarkType::kPlatform: + break; + case MarkType::kPrivate: + break; + case MarkType::kReturn: + this->mdHeaderOut(3); + fprintf(fOut, "Return Value"); + this->lf(2); + break; + case MarkType::kRow: + if (fInList) { + fprintf(fOut, " <tr>"); + this->lf(1); + } + break; + case MarkType::kSeeAlso: + this->mdHeaderOut(3); + fprintf(fOut, "See Also"); + this->lf(2); + break; + case MarkType::kStdOut: { + TextParser code(def); + this->mdHeaderOut(4); + fprintf(fOut, + "Example Output\n" + "\n" + "~~~~"); + this->lfAlways(1); + code.skipSpace(); + while (!code.eof()) { + const char* end = code.trimmedLineEnd(); + fprintf(fOut, "%.*s\n", (int) (end - code.fChar), code.fChar); + code.skipToLineStart(); + } + fprintf(fOut, "~~~~"); + this->lf(2); + } break; + case MarkType::kStruct: + fRoot = def->asRoot(); + this->mdHeaderOut(1); + fprintf(fOut, "<a name=\"%s\"></a> Struct %s", def->fName.c_str(), def->fName.c_str()); + this->lf(1); + break; + case MarkType::kSubstitute: + break; + case MarkType::kSubtopic: + this->mdHeaderOut(2); + fprintf(fOut, "<a name=\"%s\"></a> %s", def->fName.c_str(), printable.c_str()); + this->lf(2); + break; + case MarkType::kTable: + this->lf(2); + break; + case MarkType::kTemplate: + break; + case MarkType::kText: + break; + case MarkType::kTime: + break; + case MarkType::kToDo: + break; + case MarkType::kTopic: + this->mdHeaderOut(1); + fprintf(fOut, "<a name=\"%s\"></a> %s", this->linkName(def).c_str(), + printable.c_str()); + this->lf(1); + break; + case MarkType::kTrack: + // don't output children + return; + case MarkType::kTypedef: + break; + case MarkType::kUnion: + break; + case MarkType::kVolatile: + break; + case MarkType::kWidth: + break; + default: + SkDebugf("fatal error: MarkType::k%s unhandled in %s()\n", + fBmhParser.fMaps[(int) def->fMarkType].fName, __func__); + SkASSERT(0); // handle everything + break; + } + this->childrenOut(def, textStart); + switch (def->fMarkType) { // post child work, at least for tables + case MarkType::kCode: + this->writePending(); + fprintf(fOut, "</pre>"); + this->lf(2); + break; + case MarkType::kColumn: + if (fInList) { + this->writePending(); + fprintf(fOut, "</td>"); + this->lf(1); + } else { + fprintf(fOut, " "); + } + break; + case MarkType::kDescription: + this->writePending(); + fprintf(fOut, "</div>"); + fInDescription = false; + break; + case MarkType::kEnum: + case MarkType::kEnumClass: + this->lfAlways(2); + break; + case MarkType::kExample: + this->writePending(); + if (fHasFiddle) { + fprintf(fOut, "</fiddle-embed></div>"); + } else { + fprintf(fOut, "</pre>"); + } + this->lf(2); + break; + case MarkType::kList: + fInList = false; + this->writePending(); + fprintf(fOut, "</table>"); + this->lf(2); + break; + case MarkType::kLegend: { + SkASSERT(def->fChildren.size() == 1); + const Definition* row = def->fChildren[0]; + SkASSERT(MarkType::kRow == row->fMarkType); + size_t columnCount = row->fChildren.size(); + SkASSERT(columnCount > 0); + this->writePending(); + for (size_t index = 0; index < columnCount; ++index) { + fprintf(fOut, "| --- "); + } + fprintf(fOut, " |"); + this->lf(1); + } break; + case MarkType::kMethod: + fMethod = nullptr; + this->lfAlways(2); + fprintf(fOut, "---"); + this->lf(2); + break; + case MarkType::kConst: + case MarkType::kParam: + SkASSERT(TableState::kColumn == fTableState); + fTableState = TableState::kRow; + this->writePending(); + fprintf(fOut, "</td>\n"); + fprintf(fOut, " </tr>"); + this->lf(1); + break; + case MarkType::kReturn: + case MarkType::kSeeAlso: + this->lf(2); + break; + case MarkType::kRow: + if (fInList) { + fprintf(fOut, " </tr>"); + } else { + fprintf(fOut, "|"); + } + this->lf(1); + break; + case MarkType::kStruct: + fRoot = fRoot->rootParent(); + break; + case MarkType::kTable: + this->lf(2); + break; + case MarkType::kPrivate: + break; + default: + break; + } +} + +void MdOut::mdHeaderOutLF(int depth, int lf) { + this->lfAlways(lf); + for (int index = 0; index < depth; ++index) { + fprintf(fOut, "#"); + } + fprintf(fOut, " "); +} + +void MdOut::resolveOut(const char* start, const char* end, BmhParser::Resolvable resolvable) { + // FIXME: this needs the markdown character present when the def was defined, + // not the last markdown character the parser would have seen... + while (fBmhParser.fMC == end[-1]) { + --end; + } + if (start >= end) { + return; + } + string resolved = this->addReferences(start, end, resolvable); + trim_end_spaces(resolved); + if (resolved.length()) { + TextParser paragraph(fFileName, &*resolved.begin(), &*resolved.end(), fLineCount); + TextParser original(fFileName, start, end, fLineCount); + while (!original.eof() && '\n' == original.peek()) { + original.next(); + } + original.skipSpace(); + while (!paragraph.eof()) { + paragraph.skipWhiteSpace(); + const char* contentStart = paragraph.fChar; + paragraph.skipToEndBracket('\n'); + ptrdiff_t lineLength = paragraph.fChar - contentStart; + if (lineLength) { + this->writePending(); + fprintf(fOut, "%.*s", (int) lineLength, contentStart); + } + int linefeeds = 0; + while (lineLength > 0 && '\n' == contentStart[--lineLength]) { + ++linefeeds; + } + if (lineLength > 0) { + this->nl(); + } + fLinefeeds += linefeeds; + if (paragraph.eof()) { + break; + } + if ('\n' == paragraph.next()) { + linefeeds = 1; + if (!paragraph.eof() && '\n' == paragraph.peek()) { + linefeeds = 2; + } + this->lf(linefeeds); + } + } +#if 0 + while (end > start && end[0] == '\n') { + fprintf(fOut, "\n"); + --end; + } +#endif + } +} diff --git a/tools/bookmaker/parserCommon.cpp b/tools/bookmaker/parserCommon.cpp new file mode 100644 index 0000000000..cb55bcb640 --- /dev/null +++ b/tools/bookmaker/parserCommon.cpp @@ -0,0 +1,51 @@ +/* + * Copyright 2017 Google Inc. + * + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include "bookmaker.h" + +bool ParserCommon::parseSetup(const char* path) { + this->reset(); + sk_sp<SkData> data = SkData::MakeFromFileName(path); + if (nullptr == data.get()) { + SkDebugf("%s missing\n", path); + return false; + } + const char* rawText = (const char*) data->data(); + bool hasCR = false; + size_t dataSize = data->size(); + for (size_t index = 0; index < dataSize; ++index) { + if ('\r' == rawText[index]) { + hasCR = true; + break; + } + } + string name(path); + if (hasCR) { + vector<char> lfOnly; + for (size_t index = 0; index < dataSize; ++index) { + char ch = rawText[index]; + if ('\r' == rawText[index]) { + ch = '\n'; + if ('\n' == rawText[index + 1]) { + ++index; + } + } + lfOnly.push_back(ch); + } + fLFOnly[name] = lfOnly; + dataSize = lfOnly.size(); + rawText = &fLFOnly[name].front(); + } + fRawData[name] = data; + fStart = rawText; + fLine = rawText; + fChar = rawText; + fEnd = rawText + dataSize; + fFileName = string(path); + fLineCount = 1; + return true; +} diff --git a/tools/bookmaker/spellCheck.cpp b/tools/bookmaker/spellCheck.cpp new file mode 100644 index 0000000000..e43a412eed --- /dev/null +++ b/tools/bookmaker/spellCheck.cpp @@ -0,0 +1,455 @@ +/* + * Copyright 2017 Google Inc. + * + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include "bookmaker.h" + +#include "SkOSFile.h" +#include "SkOSPath.h" + +/* +things to do +if cap word is beginning of sentence, add it to table as lower-case + word must have only a single initial capital + +if word is camel cased, look for :: matches on suffix + +when function crosses lines, whole thing isn't seen as a 'word' e.g., search for largeArc in path + +words in external not seen + */ +struct CheckEntry { + string fFile; + int fLine; + int fCount; +}; + +class SpellCheck : public ParserCommon { +public: + SpellCheck(const BmhParser& bmh) : ParserCommon() + , fBmhParser(bmh) { + this->reset(); + } + bool check(const char* match); + void report(); +private: + enum class TableState { + kNone, + kRow, + kColumn, + }; + + bool check(Definition* ); + bool checkable(MarkType markType); + void childCheck(const Definition* def, const char* start); + void leafCheck(const char* start, const char* end); + bool parseFromFile(const char* path) override { return true; } + void printCheck(const string& str); + + void reset() override { + INHERITED::resetCommon(); + fMethod = nullptr; + fRoot = nullptr; + fTableState = TableState::kNone; + fInCode = false; + fInConst = false; + fInDescription = false; + fInStdOut = false; + } + + void wordCheck(const string& str); + void wordCheck(ptrdiff_t len, const char* ch); + + unordered_map<string, CheckEntry> fCode; + unordered_map<string, CheckEntry> fColons; + unordered_map<string, CheckEntry> fDigits; + unordered_map<string, CheckEntry> fDots; + unordered_map<string, CheckEntry> fParens; // also hold destructors, operators + unordered_map<string, CheckEntry> fUnderscores; + unordered_map<string, CheckEntry> fWords; + const BmhParser& fBmhParser; + Definition* fMethod; + RootDefinition* fRoot; + TableState fTableState; + bool fInCode; + bool fInConst; + bool fInDescription; + bool fInStdOut; + typedef ParserCommon INHERITED; +}; + +/* This doesn't perform a traditional spell or grammar check, although + maybe it should. Instead it looks for words used uncommonly and lower + case words that match capitalized words that are not sentence starters. + It also looks for articles preceeding capitalized words and their + modifiers to try to maintain a consistent voice. + Maybe also look for passive verbs (e.g. 'is') and suggest active ones? + */ +void BmhParser::spellCheck(const char* match) const { + SpellCheck checker(*this); + checker.check(match); + checker.report(); +} + +bool SpellCheck::check(const char* match) { + for (const auto& topic : fBmhParser.fTopicMap) { + Definition* topicDef = topic.second; + if (topicDef->fParent) { + continue; + } + if (!topicDef->isRoot()) { + return this->reportError<bool>("expected root topic"); + } + fRoot = topicDef->asRoot(); + if (string::npos == fRoot->fFileName.rfind(match)) { + continue; + } + this->check(topicDef); + } + return true; +} + +bool SpellCheck::check(Definition* def) { + fFileName = def->fFileName; + fLineCount = def->fLineCount; + string printable = def->printableName(); + const char* textStart = def->fContentStart; + if (MarkType::kParam != def->fMarkType && MarkType::kConst != def->fMarkType && + TableState::kNone != fTableState) { + fTableState = TableState::kNone; + } + switch (def->fMarkType) { + case MarkType::kAlias: + break; + case MarkType::kAnchor: + break; + case MarkType::kBug: + break; + case MarkType::kClass: + this->wordCheck(def->fName); + break; + case MarkType::kCode: + fInCode = true; + break; + case MarkType::kColumn: + break; + case MarkType::kComment: + break; + case MarkType::kConst: { + fInConst = true; + if (TableState::kNone == fTableState) { + fTableState = TableState::kRow; + } + if (TableState::kRow == fTableState) { + fTableState = TableState::kColumn; + } + this->wordCheck(def->fName); + const char* lineEnd = strchr(textStart, '\n'); + this->wordCheck(lineEnd - textStart, textStart); + textStart = lineEnd; + } break; + case MarkType::kDefine: + break; + case MarkType::kDefinedBy: + break; + case MarkType::kDeprecated: + break; + case MarkType::kDescription: + fInDescription = true; + break; + case MarkType::kDoxygen: + break; + case MarkType::kEnum: + case MarkType::kEnumClass: + this->wordCheck(def->fName); + break; + case MarkType::kError: + break; + case MarkType::kExample: + break; + case MarkType::kExternal: + break; + case MarkType::kFile: + break; + case MarkType::kFormula: + break; + case MarkType::kFunction: + break; + case MarkType::kHeight: + break; + case MarkType::kImage: + break; + case MarkType::kLegend: + break; + case MarkType::kList: + break; + case MarkType::kMember: + break; + case MarkType::kMethod: { + string method_name = def->methodName(); + string formattedStr = def->formatFunction(); + if (!def->isClone()) { + this->wordCheck(method_name); + } + fTableState = TableState::kNone; + fMethod = def; + } break; + case MarkType::kParam: { + if (TableState::kNone == fTableState) { + fTableState = TableState::kRow; + } + if (TableState::kRow == fTableState) { + fTableState = TableState::kColumn; + } + TextParser paramParser(def->fFileName, def->fStart, def->fContentStart, + def->fLineCount); + paramParser.skipWhiteSpace(); + SkASSERT(paramParser.startsWith("#Param")); + paramParser.next(); // skip hash + paramParser.skipToNonAlphaNum(); // skip Param + paramParser.skipSpace(); + const char* paramName = paramParser.fChar; + paramParser.skipToSpace(); + fInCode = true; + this->wordCheck(paramParser.fChar - paramName, paramName); + fInCode = false; + } break; + case MarkType::kPlatform: + break; + case MarkType::kReturn: + break; + case MarkType::kRow: + break; + case MarkType::kSeeAlso: + break; + case MarkType::kStdOut: { + fInStdOut = true; + TextParser code(def); + code.skipSpace(); + while (!code.eof()) { + const char* end = code.trimmedLineEnd(); + this->wordCheck(end - code.fChar, code.fChar); + code.skipToLineStart(); + } + fInStdOut = false; + } break; + case MarkType::kStruct: + fRoot = def->asRoot(); + this->wordCheck(def->fName); + break; + case MarkType::kSubtopic: + this->printCheck(printable); + break; + case MarkType::kTable: + break; + case MarkType::kTemplate: + break; + case MarkType::kText: + break; + case MarkType::kTime: + break; + case MarkType::kToDo: + break; + case MarkType::kTopic: + this->printCheck(printable); + break; + case MarkType::kTrack: + // don't output children + return true; + case MarkType::kTypedef: + break; + case MarkType::kUnion: + break; + case MarkType::kWidth: + break; + default: + SkASSERT(0); // handle everything + break; + } + this->childCheck(def, textStart); + switch (def->fMarkType) { // post child work, at least for tables + case MarkType::kCode: + fInCode = false; + break; + case MarkType::kColumn: + break; + case MarkType::kDescription: + fInDescription = false; + break; + case MarkType::kEnum: + case MarkType::kEnumClass: + break; + case MarkType::kExample: + break; + case MarkType::kLegend: + break; + case MarkType::kMethod: + fMethod = nullptr; + break; + case MarkType::kConst: + fInConst = false; + case MarkType::kParam: + SkASSERT(TableState::kColumn == fTableState); + fTableState = TableState::kRow; + break; + case MarkType::kReturn: + case MarkType::kSeeAlso: + break; + case MarkType::kRow: + break; + case MarkType::kStruct: + fRoot = fRoot->rootParent(); + break; + case MarkType::kTable: + break; + default: + break; + } + return true; +} + +bool SpellCheck::checkable(MarkType markType) { + return BmhParser::Resolvable::kYes == fBmhParser.fMaps[(int) markType].fResolve; +} + +void SpellCheck::childCheck(const Definition* def, const char* start) { + const char* end; + fLineCount = def->fLineCount; + if (def->isRoot()) { + fRoot = const_cast<RootDefinition*>(def->asRoot()); + } + for (auto& child : def->fChildren) { + end = child->fStart; + if (this->checkable(def->fMarkType)) { + this->leafCheck(start, end); + } + this->check(child); + start = child->fTerminator; + } + if (this->checkable(def->fMarkType)) { + end = def->fContentEnd; + this->leafCheck(start, end); + } +} + +void SpellCheck::leafCheck(const char* start, const char* end) { + TextParser text("", start, end, fLineCount); + do { + const char* lineStart = text.fChar; + text.skipToAlpha(); + if (text.eof()) { + break; + } + const char* wordStart = text.fChar; + text.fChar = lineStart; + text.skipTo(wordStart); // advances line number + text.skipToNonAlphaNum(); + fLineCount = text.fLineCount; + string word(wordStart, text.fChar - wordStart); + wordCheck(word); + } while (!text.eof()); +} + +void SpellCheck::printCheck(const string& str) { + string word; + for (std::stringstream stream(str); stream >> word; ) { + wordCheck(word); + } +} + +void SpellCheck::report() { + for (auto iter : fWords) { + if (string::npos != iter.second.fFile.find("undocumented.bmh")) { + continue; + } + if (string::npos != iter.second.fFile.find("markup.bmh")) { + continue; + } + if (string::npos != iter.second.fFile.find("usingBookmaker.bmh")) { + continue; + } + if (iter.second.fCount == 1) { + SkDebugf("%s %s %d\n", iter.first.c_str(), iter.second.fFile.c_str(), + iter.second.fLine); + } + } +} + +void SpellCheck::wordCheck(const string& str) { + bool hasColon = false; + bool hasDot = false; + bool hasParen = false; + bool hasUnderscore = false; + bool sawDash = false; + bool sawDigit = false; + bool sawSpecial = false; + SkASSERT(str.length() > 0); + SkASSERT(isalpha(str[0]) || '~' == str[0]); + for (char ch : str) { + if (isalpha(ch) || '-' == ch) { + sawDash |= '-' == ch; + continue; + } + bool isColon = ':' == ch; + hasColon |= isColon; + bool isDot = '.' == ch; + hasDot |= isDot; + bool isParen = '(' == ch || ')' == ch || '~' == ch || '=' == ch || '!' == ch; + hasParen |= isParen; + bool isUnderscore = '_' == ch; + hasUnderscore |= isUnderscore; + if (isColon || isDot || isUnderscore || isParen) { + continue; + } + if (isdigit(ch)) { + sawDigit = true; + continue; + } + if ('&' == ch || ',' == ch || ' ' == ch) { + sawSpecial = true; + continue; + } + SkASSERT(0); + } + if (sawSpecial && !hasParen) { + SkASSERT(0); + } + bool inCode = fInCode; + if (hasUnderscore && isupper(str[0]) && ('S' != str[0] || 'K' != str[1]) + && !hasColon && !hasDot && !hasParen && !fInStdOut && !inCode && !fInConst + && !sawDigit && !sawSpecial && !sawDash) { + std::istringstream ss(str); + string token; + while (std::getline(ss, token, '_')) { + this->wordCheck(token); + } + return; + } + if (!hasColon && !hasDot && !hasParen && !hasUnderscore + && !fInStdOut && !inCode && !fInConst && !sawDigit + && islower(str[0]) && isupper(str[1])) { + inCode = true; + } + auto& mappy = hasColon ? fColons : + hasDot ? fDots : + hasParen ? fParens : + hasUnderscore ? fUnderscores : + fInStdOut || inCode || fInConst ? fCode : + sawDigit ? fDigits : fWords; + auto iter = mappy.find(str); + if (mappy.end() != iter) { + iter->second.fCount += 1; + } else { + CheckEntry* entry = &mappy[str]; + entry->fFile = fFileName; + entry->fLine = fLineCount; + entry->fCount = 1; + } +} + +void SpellCheck::wordCheck(ptrdiff_t len, const char* ch) { + leafCheck(ch, ch + len); +} |