multi language open tasks

README.md

@@ -46,6 +46,7 @@ All supported languages and their supported versions are listed below.
 | [EMF Model](https://www.eclipse.org/modeling/emf/)     |                                                                                 2.25.0 | emf-model         |                                alpha                                |    EMF    |
 | [SCXML](https://www.w3.org/TR/scxml/)                  |                                                                                    1.0 | scxml             |                                alpha                                |    XML    |
 | Text (naive, use with caution)                         |                                                                                      - | text              |                               legacy                                |  CoreNLP  |
+| Multi-Language                                         |                                                                                      - | multi             |                                alpha                                |     -     |
 
 ## Download and Installation
 You need Java SE 21 to run or build JPlag.

language-api/src/main/java/de/jplag/Language.java

@@ -122,4 +122,18 @@ default boolean supportsNormalization() {
     default boolean requiresCoreNormalization() {
         return true;
     }
+
+    /**
+     * @return True, if the language module can be used by the multi-language module
+     */
+    default boolean supportsMultilanguage() {
+        return true;
+    }
+
+    /**
+     * @return True, if the language module should be used in case of ambiguities
+     */
+    default boolean hasPriority() {
+        return false;
+    }
 }

languages/cpp/src/main/java/de/jplag/cpp/CPPLanguage.java

@@ -51,4 +51,9 @@ public boolean supportsNormalization() {
     public List<Token> parse(Set<File> files, boolean normalize) throws ParsingException {
         return new CPPParserAdapter().parse(files);
     }
+
+    @Override
+    public boolean hasPriority() {
+        return true;
+    }
 }

languages/emf-model/src/main/java/de/jplag/emf/model/EmfModelLanguage.java

@@ -55,4 +55,9 @@ public List<File> customizeSubmissionOrder(List<File> sub) {
     public List<Token> parse(Set<File> files, boolean normalize) throws ParsingException {
         return new DynamicModelParser().parse(files, normalize);
     }
+
+    @Override
+    public boolean supportsMultilanguage() {
+        return false;
+    }
 }

languages/multi-language/README.md

@@ -0,0 +1,7 @@
+# Multi-Language module
+
+This module allows parsing multiple languages in one run of JPlag. The parsing will be delegated to a different language-module per file.
+
+This does not entail comparing implementations of the same functionality in different languages to each other, but allows comparing submissions that use more than one language each.
+
+By default, all supported languages (except text) are parsed. This can be changed by specifying the language modules to use with the --languages parameter: `java -jar jplag.jar <root folder> multi --languages <firstLanguage>,<secondLanguage>,...`

languages/multi-language/src/main/java/de/jplag/multilang/MultiLanguageOptions.java

@@ -20,14 +20,14 @@ public class MultiLanguageOptions extends LanguageOptions {
     public List<Language> getLanguages() {
         if (this.languages == null) {
             if (languageNames.getValue() == null) {
-                throw new IllegalArgumentException(ERROR_NOT_ENOUGH_LANGUAGES);
+                this.languages = LanguageLoader.getAllAvailableLanguages().values().stream().filter(Language::supportsMultilanguage).toList();
+            } else {
+                this.languages = Arrays.stream(languageNames.getValue().split(","))
+                        .map(name -> LanguageLoader.getLanguage(name)
+                                .orElseThrow(() -> new IllegalArgumentException(String.format(ERROR_LANGUAGE_NOT_FOUND, name))))
+                        .filter(language -> !language.getClass().equals(MultiLanguage.class)).toList();
             }
 
-            this.languages = Arrays.stream(languageNames.getValue().split(","))
-                    .map(name -> LanguageLoader.getLanguage(name)
-                            .orElseThrow(() -> new IllegalArgumentException(String.format(ERROR_LANGUAGE_NOT_FOUND, name))))
-                    .filter(language -> !language.getClass().equals(MultiLanguage.class)).toList();
-
             if (this.languages.isEmpty()) {
                 throw new IllegalArgumentException(ERROR_NOT_ENOUGH_LANGUAGES);
             }

languages/multi-language/src/main/java/de/jplag/multilang/MultiLanguageParser.java

@@ -7,18 +7,28 @@
 import java.util.Optional;
 import java.util.Set;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import de.jplag.Language;
 import de.jplag.ParsingException;
 import de.jplag.Token;
 
 public class MultiLanguageParser {
+    private static final Logger LOG = LoggerFactory.getLogger(MultiLanguageParser.class);
+    private static final String WARNING = "This module only allows parsing of multiple languages. No comparisons will be made between languages";
+    private static final String ERROR_MULTIPLE_PRIORITY_LANGUAGES = "Multiple language modules with priority (%s) have been found for file: %s";
+    private static final String ERROR_MULTIPLE_LANGUAGES = "Multiple language modules (%s) have been found for file: %s";
     private final List<Language> languages;
+    private static boolean hasPrintedWarning;
 
     public MultiLanguageParser(MultiLanguageOptions options) {
         this.languages = options.getLanguages();
+        hasPrintedWarning = false;
     }
 
     public List<Token> parseFiles(Set<File> files, boolean normalize) throws ParsingException {
+        this.printWarning();
         List<Token> results = new ArrayList<>();
         for (File file : files) {
             Optional<Language> language = findLanguageForFile(file);
@@ -29,8 +39,45 @@ public List<Token> parseFiles(Set<File> files, boolean normalize) throws Parsing
         return results;
     }
 
-    private Optional<Language> findLanguageForFile(File file) {
-        return this.languages.stream().filter(language -> Arrays.stream(language.suffixes()).anyMatch(suffix -> file.getName().endsWith(suffix)))
-                .findFirst();
+    private Optional<Language> findLanguageForFile(File file) throws ParsingException {
+        List<Language> normalLanguages = new ArrayList<>();
+        List<Language> priorityLanguages = new ArrayList<>();
+
+        for (Language language : this.languages) {
+            if (Arrays.stream(language.suffixes()).anyMatch(it -> file.getName().toLowerCase().endsWith(it.toLowerCase()))) {
+                if (language.hasPriority()) {
+                    priorityLanguages.add(language);
+                } else {
+                    normalLanguages.add(language);
+                }
+            }
+        }
+
+        if (!priorityLanguages.isEmpty()) {
+            if (priorityLanguages.size() > 1) {
+                throw new ParsingException(file, String.format(ERROR_MULTIPLE_PRIORITY_LANGUAGES,
+                        String.join(", ", priorityLanguages.stream().map(Language::getName).toList()), file.getPath()));
+            }
+
+            return Optional.of(priorityLanguages.getFirst());
+        } else {
+            if (normalLanguages.isEmpty()) {
+                return Optional.empty();
+            }
+
+            if (normalLanguages.size() > 1) {
+                throw new ParsingException(file, String.format(ERROR_MULTIPLE_LANGUAGES,
+                        String.join(", ", normalLanguages.stream().map(Language::getName).toList()), file.getPath()));
+            }
+
+            return Optional.of(normalLanguages.getFirst());
+        }
+    }
+
+    private void printWarning() {
+        if (!hasPrintedWarning) {
+            hasPrintedWarning = true;
+            LOG.warn(WARNING);
+        }
     }
 }

languages/multi-language/src/test/java/de/java/multilang/MultilangTest.java

@@ -52,14 +52,6 @@ void testMultiLanguageParsing() throws ParsingException {
         Assertions.assertEquals(expectedTokens, tokens.stream().map(Token::getType).toList());
     }
 
-    @Test
-    void testNoLanguagesConfigured() {
-        MultiLanguage languageModule = new MultiLanguage();
-        Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> {
-            languageModule.parse(Set.of(javaCode, cppCode), false);
-        });
-    }
-
     @Test
     void testInvalidLanguage() {
         MultiLanguage languageModule = new MultiLanguage();

languages/text/src/main/java/de/jplag/text/NaturalLanguage.java

@@ -43,4 +43,9 @@ public int minimumTokenMatch() {
     public List<Token> parse(Set<File> files, boolean normalize) throws ParsingException {
         return new ParserAdapter().parse(files);
     }
+
+    @Override
+    public boolean supportsMultilanguage() {
+        return false;
+    }
 }

languages/typescript/src/main/java/de/jplag/typescript/TypeScriptLanguage.java

@@ -47,4 +47,9 @@ public TypeScriptLanguageOptions getOptions() {
     public List<Token> parse(Set<File> files, boolean normalize) throws ParsingException {
         return new TypeScriptParserAdapter(options.useStrictDefault()).parse(files);
     }
+
+    @Override
+    public boolean hasPriority() {
+        return true;
+    }
 }