Skip to content

Commit 9b4e9f8

Browse files
SpartanJSpartanJ
committed
TextMate parsing fixes and improvements.
1 parent 132dd01 commit 9b4e9f8

File tree

4 files changed

+60
-33
lines changed

4 files changed

+60
-33
lines changed

‎include/eepp/ui/doc/syntaxdefinition.hpp

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -138,11 +138,12 @@ class EE_API SyntaxDefinition {
138138
public:
139139
SyntaxDefinition();
140140

141-
SyntaxDefinition( const std::string& languageName, std::vector<std::string>&& files,
142-
std::vector<SyntaxPattern>&& patterns,
143-
SyntaxDefMap<std::string, std::string>&& symbols = {},
144-
const std::string& comment = "", std::vector<std::string>&& headers = {},
145-
const std::string& lspName = "" );
141+
SyntaxDefinition(
142+
const std::string& languageName, std::vector<std::string>&& files,
143+
std::vector<SyntaxPattern>&& patterns,
144+
SyntaxDefMap<std::string, std::string>&& symbols = {}, const std::string& comment = "",
145+
std::vector<std::string>&& headers = {}, const std::string& lspName = "",
146+
std::vector<std::pair<std::string, std::vector<SyntaxPattern>>>&& repositories = {} );
146147

147148
const std::string& getLanguageName() const;
148149

@@ -230,8 +231,10 @@ class EE_API SyntaxDefinition {
230231

231232
SyntaxDefinition& setFoldBraces( const std::vector<std::pair<Int64, Int64>>& foldBraces );
232233

233-
SyntaxDefinition& addRepository( const std::string& name,
234-
std::vector<SyntaxPattern>&& patterns );
234+
SyntaxDefinition& addRepository( std::string&& name, std::vector<SyntaxPattern>&& patterns );
235+
236+
SyntaxDefinition& addRepositories(
237+
std::vector<std::pair<std::string, std::vector<SyntaxPattern>>>&& repositories );
235238

236239
const std::vector<SyntaxPattern>& getRepository( String::HashType hash ) const;
237240

‎src/eepp/ui/doc/syntaxdefinition.cpp

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@ static void liftContentPatternsRecursive( SyntaxDefinition& def, SyntaxPattern&
2626
std::string contentRepoName = "$CONTENT_" + def.getLanguageNameForFileSystem() + "_" +
2727
namePrefixSeed + "_uid" +
2828
String::toString( uniqueIdCounter++ );
29-
def.addRepository( contentRepoName, std::move( pattern.contentPatterns ) );
3029
pattern.contentScopeRepoHash = String::hash( contentRepoName );
30+
def.addRepository( std::move( contentRepoName ), std::move( pattern.contentPatterns ) );
3131
}
3232
}
3333

@@ -99,12 +99,11 @@ static void updateRepoIndexState( SyntaxDefinition& def, std::vector<SyntaxPatte
9999

100100
SyntaxDefinition::SyntaxDefinition() {}
101101

102-
SyntaxDefinition::SyntaxDefinition( const std::string& languageName,
103-
std::vector<std::string>&& files,
104-
std::vector<SyntaxPattern>&& patterns,
105-
SyntaxDefMap<std::string, std::string>&& symbols,
106-
const std::string& comment, std::vector<std::string>&& headers,
107-
const std::string& lspName ) :
102+
SyntaxDefinition::SyntaxDefinition(
103+
const std::string& languageName, std::vector<std::string>&& files,
104+
std::vector<SyntaxPattern>&& patterns, SyntaxDefMap<std::string, std::string>&& symbols,
105+
const std::string& comment, std::vector<std::string>&& headers, const std::string& lspName,
106+
std::vector<std::pair<std::string, std::vector<SyntaxPattern>>>&& repositories ) :
108107
mLanguageName( languageName ),
109108
mLanguageId( String::hash( String::toLower( languageName ) ) ),
110109
mFiles( std::move( files ) ),
@@ -126,6 +125,7 @@ SyntaxDefinition::SyntaxDefinition( const std::string& languageName,
126125
}
127126
for ( const auto& symbol : mSymbolNames )
128127
mSymbols.insert( { symbol.first, toSyntaxStyleType( symbol.second ) } );
128+
addRepositories( std::move( repositories ) );
129129
}
130130

131131
const std::vector<std::string>& SyntaxDefinition::getFiles() const {
@@ -435,18 +435,29 @@ SyntaxPattern::SyntaxPattern( std::vector<std::string>&& _patterns,
435435
updateCache<SyntaxStyleType>( *this );
436436
}
437437

438-
SyntaxDefinition& SyntaxDefinition::addRepository( const std::string& name,
438+
SyntaxDefinition& SyntaxDefinition::addRepository( std::string&& name,
439439
std::vector<SyntaxPattern>&& patterns ) {
440+
440441
eeASSERT( patterns.size() < std::numeric_limits<Uint8>::max() - 1 );
441442
auto hash = String::hash( name );
442443
mRepositoryIndex[hash] = ++mRepositoryIndexCounter;
443-
mRepositoryNames[hash] = name;
444444
mRepositoryIndexInvert[mRepositoryIndexCounter] = hash;
445445
for ( auto& ptrn : patterns )
446446
ptrn.repositoryIdx = mRepositoryIndexCounter;
447447
updatePatternsState( *this, patterns );
448448
mRepository[hash] = std::move( patterns );
449449
updateRepoIndexState( *this, mPatterns );
450+
mRepositoryNames[hash] = std::move( name );
451+
return *this;
452+
}
453+
454+
SyntaxDefinition& SyntaxDefinition::addRepositories(
455+
std::vector<std::pair<std::string, std::vector<SyntaxPattern>>>&& repositories ) {
456+
if ( !repositories.empty() ) {
457+
for ( auto& ptrn : repositories )
458+
addRepository( std::move( ptrn.first ), std::move( ptrn.second ) );
459+
compile();
460+
}
450461
return *this;
451462
}
452463

‎src/eepp/ui/doc/syntaxdefinitionmanager.cpp

Lines changed: 30 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
#include <eepp/ui/doc/languages/python.hpp>
1717
#include <eepp/ui/doc/languages/xml.hpp>
1818
#include <eepp/ui/doc/syntaxdefinitionmanager.hpp>
19+
1920
#include <nlohmann/json.hpp>
2021

2122
using namespace EE::System;
@@ -401,7 +402,7 @@ static void patternToCPP( std::string& buf, const SyntaxPattern& pattern,
401402
}
402403
if ( pattern.hasContentScope() ) {
403404
const auto& patterns = def.getRepository( pattern.contentScopeRepoHash );
404-
buf += "{\n";
405+
buf += ", {\n";
405406
for ( const auto& ptrn : patterns )
406407
patternToCPP( buf, ptrn, def );
407408
buf += "\n}";
@@ -452,6 +453,7 @@ namespace EE { namespace UI { namespace Doc { namespace Language {
452453
buf += join( def.getHeaders() ) + ( lspName.empty() ? "" : "," ) + "\n";
453454
// lsp
454455
buf += lspName.empty() ? "" : str( def.getLSPName() );
456+
455457
buf += "\n}";
456458
buf += ")";
457459
if ( !def.isVisible() )
@@ -485,21 +487,22 @@ namespace EE { namespace UI { namespace Doc { namespace Language {
485487
String( static_cast<String::StringBaseType>( brace.first ) ).toUtf8(),
486488
String( static_cast<String::StringBaseType>( brace.second ) ).toUtf8() );
487489
}
488-
buf += " } );";
490+
buf += " } )";
489491
}
490492

491-
for ( const auto& repo : def.getRepositories() ) {
492-
std::string name = def.getRepositoryName( repo.first );
493-
if ( name.starts_with( "$CONTENT_" ) )
494-
continue;
495-
buf += ".addRepository( \"" + name + "\",\n\t\t";
496-
patternsToCPP( buf, repo.second );
497-
buf += "\n)";
493+
if ( !def.getRepositories().empty() ) {
494+
buf += ".addRepositories( {\n";
495+
for ( const auto& repo : def.getRepositories() ) {
496+
std::string name = def.getRepositoryName( repo.first );
497+
if ( name.starts_with( "$CONTENT_" ) )
498+
continue;
499+
buf += "\n{ \"" + name + "\", ";
500+
patternsToCPP( buf, repo.second );
501+
buf += "\n}, ";
502+
}
503+
buf += "\n} )";
498504
}
499505

500-
if ( !def.getRepositories().empty() )
501-
buf += ".compile()";
502-
503506
buf += ";\n}\n";
504507
buf += "\n}}}} // namespace EE::UI::Doc::Language\n";
505508
return std::make_pair( std::move( header ), std::move( buf ) );
@@ -765,6 +768,19 @@ static SyntaxPattern parsePattern( const nlohmann::json& pattern ) {
765768
ptrns.emplace_back( pattern["parser"] );
766769
}
767770
}
771+
772+
// Sub-languages / Sub patterns?
773+
if ( pattern.contains( "patterns" ) && !pattern["patterns"].empty() &&
774+
pattern["patterns"].is_array() ) {
775+
const auto& patterns = pattern["patterns"];
776+
subPatterns.reserve( patterns.size() );
777+
for ( const auto& subPattern : patterns )
778+
subPatterns.push_back( parsePattern( subPattern ) );
779+
}
780+
781+
if ( type.empty() && ( pattern.contains( "name" ) || pattern.contains( "begin" ) ) ) {
782+
type.emplace_back( "normal" );
783+
}
768784
}
769785

770786
eeASSERT( !ptrns.empty() );
@@ -816,7 +832,8 @@ static SyntaxDefinition loadLanguage( const nlohmann::json& json ) {
816832
for ( const auto& pattern : repository )
817833
ptrns.emplace_back( parsePattern( pattern ) );
818834
}
819-
def.addRepository( name, std::move( ptrns ) );
835+
auto rname( name );
836+
def.addRepository( std::move( rname ), std::move( ptrns ) );
820837
}
821838
}
822839

‎src/eepp/ui/doc/syntaxhighlighter.cpp

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -131,10 +131,6 @@ void SyntaxHighlighter::setMaxTokenizationLength( const Int64& maxTokenizationLe
131131

132132
void SyntaxHighlighter::tokenizeAsync( std::shared_ptr<ThreadPool> pool,
133133
const std::function<void()>& onDone ) {
134-
#if !defined( EE_COMPILER_MSVC ) && defined( EE_DEBUG )
135-
#warning SyntaxHighlighter::tokenizeAsync is disabled
136-
return;
137-
#endif
138134
if ( mTokenizeAsync )
139135
return;
140136
mTokenizeAsync = true;

0 commit comments

Comments
 (0)