Refactor syntax tokenizer to simplify future improvements for it.

Author: SpartanJ

include/eepp/ui/doc/syntaxdefinition.hpp

@@ -28,6 +28,8 @@ template <typename T> static auto toSyntaxStyleTypeV( const std::vector<T>& s )
 
 enum class SyntaxPatternMatchType { LuaPattern, RegEx, Parser };
 
+class SyntaxDefinition;
+
 struct EE_API SyntaxPattern {
 	static UnorderedMap<SyntaxStyleType, std::string> SyntaxStyleTypeCache;
 
@@ -42,6 +44,7 @@ struct EE_API SyntaxPattern {
 	std::string syntax{ "" };
 	DynamicSyntax dynSyntax;
 	SyntaxPatternMatchType matchType{ SyntaxPatternMatchType::LuaPattern };
+	const SyntaxDefinition* def{ nullptr };
 
 	SyntaxPattern( std::vector<std::string>&& _patterns, const std::string& _type,
 				   const std::string& _syntax = "",
@@ -67,6 +70,17 @@ struct EE_API SyntaxPattern {
 				   std::vector<std::string>&& _endTypes, DynamicSyntax&& _syntax,
 				   SyntaxPatternMatchType matchType = SyntaxPatternMatchType::LuaPattern );
 
+	bool isIncludePattern() const {
+		return patterns.size() == 2 && patterns[0] == "include" && !patterns[1].empty() &&
+			   ( patterns[1][0] == '#' || patterns[1][0] == '$' );
+	}
+
+	bool isRangedMatch() const { return !isIncludePattern() && patterns.size() >= 2; }
+
+	bool isRootSelfInclude() const { return isIncludePattern() && patterns[1] == "$self"; }
+
+	bool isRepositoryInclude() const { return isIncludePattern() && patterns[1][0] == '#'; }
+
 	bool hasSyntax() const { return !syntax.empty() || dynSyntax; }
 };
 

src/eepp/ui/doc/syntaxdefinition.cpp

@@ -31,6 +31,17 @@ template <typename SyntaxStyleType> void updateCache( const SyntaxPattern& ptrn
 	}
 }
 
+static void updatePatternRefs( const SyntaxDefinition& def, SyntaxPattern& ptrn ) {
+	ptrn.def = &def;
+	if ( ptrn.syntax == "$self" )
+		ptrn.syntax = def.getLanguageName();
+}
+
+static void updatePatternRefs( const SyntaxDefinition& def, std::vector<SyntaxPattern>& ptrns ) {
+	for ( auto& ptrn : ptrns )
+		updatePatternRefs( def, ptrn );
+}
+
 SyntaxDefinition::SyntaxDefinition() {}
 
 SyntaxDefinition::SyntaxDefinition( const std::string& languageName,
@@ -54,6 +65,7 @@ SyntaxDefinition::SyntaxDefinition( const std::string& languageName,
 				 return pattern.matchType == SyntaxPatternMatchType::Parser;
 			 } ) )
 			ParserMatcherManager::instance()->registerBaseParsers();
+		updatePatternRefs( *this, mPatterns );
 		mPatterns.emplace_back( SyntaxPattern{ { "%s+" }, "normal" } );
 		mPatterns.emplace_back( SyntaxPattern{ { "%w+%f[%s]" }, "normal" } );
 	}
@@ -348,6 +360,7 @@ SyntaxPattern::SyntaxPattern( std::vector<std::string>&& _patterns,
 
 SyntaxDefinition& SyntaxDefinition::setRepository( const std::string& name,
 												   std::vector<SyntaxPattern>&& patterns ) {
+	updatePatternRefs( *this, patterns );
 	mRepository[name] = std::move( patterns );
 	return *this;
 }