Skip to content

Commit 0fbebf6

Browse files
SpartanJSpartanJ
committed
TextMate support WIP.
1 parent 3560a2d commit 0fbebf6

File tree

3 files changed

+80
-43
lines changed

3 files changed

+80
-43
lines changed

‎include/eepp/ui/doc/syntaxdefinition.hpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,10 @@ struct EE_API SyntaxPattern {
105105

106106
inline bool isRangedMatch() const { return flags & Flags::IsRangedMatch; }
107107

108+
inline bool isSimpleRangedMatch() const {
109+
return isRangedMatch() && !hasContentScope() && !hasSyntax();
110+
}
111+
108112
std::string_view getRepositoryName() const {
109113
eeASSERT( isRepositoryInclude() );
110114
return std::string_view{ patterns[1] }.substr( 1 );

‎src/eepp/ui/doc/syntaxhighlighter.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@ void SyntaxHighlighter::setMaxTokenizationLength( const Int64& maxTokenizationLe
131131

132132
void SyntaxHighlighter::tokenizeAsync( std::shared_ptr<ThreadPool> pool,
133133
const std::function<void()>& onDone ) {
134+
#warning SyntaxHighlighter::tokenizeAsync is disabled
134135
return;
135136
if ( mTokenizeAsync )
136137
return;

‎src/eepp/ui/doc/syntaxtokenizer.cpp

Lines changed: 75 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -156,21 +156,26 @@ SyntaxStateRestored SyntaxTokenizer::retrieveSyntaxState( const SyntaxDefinition
156156
if ( state.state[0].state > 0 &&
157157
( state.state[1].state > 0 ||
158158
( ( curPattern = syntax.getPatternFromState( state.state[0] ) ) &&
159-
curPattern->hasSyntax() ) ) ) {
159+
curPattern->hasSyntaxOrContentScope() ) ) ) {
160160
for ( size_t i = 0; i < MAX_SUB_SYNTAXS - 1; ++i ) {
161161
if ( i != 0 || curPattern == nullptr )
162162
curPattern = syntaxState.currentSyntax->getPatternFromState( state.state[i] );
163163
if ( curPattern && state.state[i].state != SYNTAX_TOKENIZER_STATE_NONE ) {
164-
if ( curPattern->hasSyntax() ) {
164+
if ( curPattern->hasSyntaxOrContentScope() ) {
165165
syntaxState.subsyntaxInfo = curPattern;
166166
auto langIndex = state.langStack[i];
167167
syntaxState.currentSyntax =
168168
langIndex != 0
169169
? &SyntaxDefinitionManager::instance()->getByLanguageIndex( langIndex )
170170
: &SyntaxDefinitionManager::instance()->getByLanguageName(
171171
syntaxState.subsyntaxInfo->syntax );
172-
syntaxState.currentPatternIdx = {};
173-
syntaxState.currentLevel++;
172+
if ( curPattern->hasContentScope() ) {
173+
syntaxState.currentPatternIdx = state.state[i];
174+
syntaxState.currentLevel = i;
175+
} else {
176+
syntaxState.currentPatternIdx = {};
177+
syntaxState.currentLevel++;
178+
}
174179
} else {
175180
syntaxState.currentPatternIdx = state.state[i];
176181
}
@@ -186,33 +191,65 @@ static inline void setSubsyntaxPatternIdx( SyntaxStateRestored& curState, Syntax
186191
const SyntaxStateType& patternIndex ) {
187192
curState.currentPatternIdx = patternIndex;
188193
retState.state[curState.currentLevel] = patternIndex;
189-
};
194+
}
190195

191-
static inline void pushSubsyntax( SyntaxStateRestored& curState, SyntaxState& retState,
192-
const SyntaxPattern& enteringSubsyntax,
193-
const SyntaxStateType& patternIndex,
194-
std::string_view patternTextStr ) {
196+
static inline void pushStack( SyntaxStateRestored& curState, SyntaxState& retState,
197+
const SyntaxPattern& enteringSubsyntax,
198+
const SyntaxStateType& patternIndex,
199+
std::string_view patternTextStr ) {
195200
if ( curState.currentLevel == MAX_SUB_SYNTAXS - 1 )
196201
return;
202+
203+
if ( !enteringSubsyntax.hasSyntax() ) {
204+
if ( retState.langStack[curState.currentLevel] != 0 ||
205+
retState.state[curState.currentLevel].state != 0 )
206+
curState.currentLevel++;
207+
curState.subsyntaxInfo = &enteringSubsyntax;
208+
retState.langStack[curState.currentLevel] = curState.currentSyntax->getLanguageIndex();
209+
setSubsyntaxPatternIdx( curState, retState, patternIndex );
210+
return;
211+
}
212+
197213
setSubsyntaxPatternIdx( curState, retState, patternIndex );
214+
198215
curState.subsyntaxInfo = &enteringSubsyntax;
199-
curState.currentSyntax = &SyntaxDefinitionManager::instance()->getByLanguageName(
200-
curState.subsyntaxInfo->dynSyntax
201-
? curState.subsyntaxInfo->dynSyntax( enteringSubsyntax, patternTextStr )
202-
: curState.subsyntaxInfo->syntax );
216+
curState.currentSyntax =
217+
!enteringSubsyntax.hasSyntax()
218+
? curState.currentSyntax
219+
: ( &SyntaxDefinitionManager::instance()->getByLanguageName(
220+
curState.subsyntaxInfo->dynSyntax
221+
? curState.subsyntaxInfo->dynSyntax( enteringSubsyntax, patternTextStr )
222+
: curState.subsyntaxInfo->syntax ) );
223+
203224
retState.langStack[curState.currentLevel] = curState.currentSyntax->getLanguageIndex();
204225
curState.currentLevel++;
226+
205227
setSubsyntaxPatternIdx( curState, retState, SyntaxStateType{} );
206-
};
228+
}
229+
230+
static inline void popStack( SyntaxStateRestored& curState, SyntaxState& retState,
231+
const SyntaxDefinition& syntax, const SyntaxPattern& fromPattern ) {
232+
if ( curState.currentLevel == 0 && retState.state[0].state == SYNTAX_TOKENIZER_STATE_NONE ) {
233+
Log::debug( "Attempted to pop base stack level or already at an empty base." );
234+
return;
235+
}
207236

208-
static inline void popSubsyntax( SyntaxStateRestored& curState, SyntaxState& retState,
209-
const SyntaxDefinition& syntax ) {
210237
setSubsyntaxPatternIdx( curState, retState, SyntaxStateType{} );
238+
239+
if ( fromPattern.isSimpleRangedMatch() )
240+
return;
241+
211242
retState.langStack[curState.currentLevel] = 0;
212-
curState.currentLevel--;
213-
setSubsyntaxPatternIdx( curState, retState, SyntaxStateType{} );
243+
if ( curState.currentLevel > 0 )
244+
curState.currentLevel--;
245+
246+
if ( retState.langStack[curState.currentLevel] != 0 &&
247+
retState.langStack[curState.currentLevel] != syntax.getLanguageIndex() ) {
248+
setSubsyntaxPatternIdx( curState, retState, SyntaxStateType{} );
249+
}
250+
214251
curState = SyntaxTokenizer::retrieveSyntaxState( syntax, retState );
215-
};
252+
}
216253

217254
template <typename T>
218255
static inline void pushTokensToOpenCloseSubsyntax( int i, std::string_view textv,
@@ -313,7 +350,7 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
313350
pushTokensToOpenCloseSubsyntax( startIdx, textv, curState.subsyntaxInfo,
314351
*shouldCloseSubSyntax, tokens );
315352
}
316-
popSubsyntax( curState, retState, syntax );
353+
popStack( curState, retState, syntax, *curState.subsyntaxInfo );
317354
startIdx = shouldCloseSubSyntax->range.second;
318355
shouldCloseSubSyntax = {};
319356
return true;
@@ -385,18 +422,15 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
385422
}
386423
}
387424

388-
if ( !( skipSubSyntaxSeparator && pattern.hasSyntax() ) )
425+
if ( !( skipSubSyntaxSeparator && pattern.hasSyntaxOrContentScope() ) )
389426
pushToken( tokens, currentType, segmentText );
390427

391428
currentBytePos = segmentEndBytePos;
392429
}
393430

394-
if ( pattern.hasSyntax() ) {
395-
pushSubsyntax(
396-
curState, retState, pattern, patternIndex,
397-
textv.substr( fullMatchStart, fullMatchEnd - fullMatchStart ) );
398-
} else if ( pattern.patterns.size() > 1 ) {
399-
setSubsyntaxPatternIdx( curState, retState, patternIndex );
431+
if ( pattern.isRangedMatch() ) {
432+
pushStack( curState, retState, pattern, patternIndex,
433+
textv.substr( fullMatchStart, fullMatchEnd - fullMatchStart ) );
400434
}
401435

402436
startIdx = fullMatchEnd;
@@ -438,7 +472,7 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
438472
( patternTextStr = patternText ) )
439473
: SyntaxStyleEmpty();
440474

441-
if ( !skipSubSyntaxSeparator || !pattern.hasSyntax() ) {
475+
if ( !skipSubSyntaxSeparator || !pattern.hasSyntaxOrContentScope() ) {
442476
pushToken( tokens,
443477
type == SyntaxStyleEmpty()
444478
? ( curMatch < pattern.types.size()
@@ -448,13 +482,11 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
448482
patternText );
449483
}
450484

451-
if ( pattern.hasSyntax() && curMatch == numMatches - 1 &&
485+
if ( pattern.isRangedMatch() && curMatch == numMatches - 1 &&
452486
end == fullMatchEnd ) {
453-
pushSubsyntax(
487+
pushStack(
454488
curState, retState, pattern, patternIndex,
455489
textv.substr( fullMatchStart, fullMatchEnd - fullMatchStart ) );
456-
} else if ( pattern.patterns.size() > 1 ) {
457-
setSubsyntaxPatternIdx( curState, retState, patternIndex );
458490
}
459491

460492
startIdx = end;
@@ -464,8 +496,8 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
464496
textv.substr( end, fullMatchEnd - end ) );
465497
startIdx = fullMatchEnd;
466498

467-
if ( pattern.hasSyntax() && curMatch == numMatches - 1 ) {
468-
pushSubsyntax(
499+
if ( pattern.isRangedMatch() && curMatch == numMatches - 1 ) {
500+
pushStack(
469501
curState, retState, pattern, patternIndex,
470502
textv.substr( fullMatchStart, fullMatchEnd - fullMatchStart ) );
471503
}
@@ -503,15 +535,15 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
503535
? curState.currentSyntax->getSymbol( ( patternTextStr = patternText ) )
504536
: SyntaxStyleEmpty();
505537

506-
if ( !skipSubSyntaxSeparator || !pattern.hasSyntax() ) {
538+
if ( !skipSubSyntaxSeparator || !pattern.hasSyntaxOrContentScope() ) {
507539
pushToken( tokens, type == SyntaxStyleEmpty() ? pattern.types[0] : type,
508540
patternText );
509541
}
510-
if ( pattern.hasSyntax() ) {
511-
pushSubsyntax( curState, retState, pattern, patternIndex, patternText );
512-
} else if ( pattern.patterns.size() > 1 ) {
513-
setSubsyntaxPatternIdx( curState, retState, patternIndex );
542+
543+
if ( pattern.isRangedMatch() ) {
544+
pushStack( curState, retState, pattern, patternIndex, patternText );
514545
}
546+
515547
startIdx = end;
516548
return true;
517549
}
@@ -552,7 +584,7 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
552584
if ( endRange.range.first == static_cast<Int64>( startIdx ) ) {
553585
pushTokensToOpenCloseSubsyntax( startIdx, textv, activePattern, endRange,
554586
tokens, true );
555-
setSubsyntaxPatternIdx( curState, retState, SyntaxStateType{} );
587+
popStack( curState, retState, syntax, *activePattern );
556588
startIdx = endRange.range.second;
557589
continue;
558590
}
@@ -646,7 +678,7 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
646678
pushTokensToOpenCloseSubsyntax( startIdx, textv, curState.subsyntaxInfo,
647679
rangeSubsyntax, tokens, true );
648680
}
649-
popSubsyntax( curState, retState, syntax );
681+
popStack( curState, retState, syntax, *curState.subsyntaxInfo );
650682
startIdx = rangeSubsyntax.range.second;
651683
skip = true;
652684
}
@@ -656,7 +688,7 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
656688
if ( endRange.range.first != -1 ) {
657689
pushTokensToOpenCloseSubsyntax( startIdx, textv, activePattern, endRange,
658690
tokens, true );
659-
setSubsyntaxPatternIdx( curState, retState, SyntaxStateType{} );
691+
popStack( curState, retState, syntax, *activePattern );
660692
startIdx = endRange.range.second;
661693
continue;
662694
} else {
@@ -722,7 +754,7 @@ _tokenize( const SyntaxDefinition& syntax, const std::string& text, const Syntax
722754
pushTokensToOpenCloseSubsyntax( startIdx, textv, curState.subsyntaxInfo,
723755
*shouldCloseSubSyntax, tokens );
724756
}
725-
popSubsyntax( curState, retState, syntax );
757+
popStack( curState, retState, syntax, *curState.subsyntaxInfo );
726758
startIdx = shouldCloseSubSyntax->range.second;
727759
matched = true;
728760
shouldCloseSubSyntax = {};

0 commit comments

Comments
 (0)