Skip to content

Commit

Permalink
Fix recursive localization loading when using mods
Browse files Browse the repository at this point in the history 
In CK3 and some other games, the localization folder has a deep hierarchy of folders. The previous logic used FilenameResolver.listFiles() and then recursively listed files on any directories that appeared in the list. But this is incorrect behavior if the same subfolder appears in both places, as listFiles() will believe the subfolder in the mod overrides the subfolder in the main. In reality, both subfolders need to be considered, and only identical filenames *inside* the folder are overriden.
  • Loading branch information
@mmyers
mmyers committed Jun 20, 2024
1 parent 1bfcdd4 commit a142282
Show file tree
Hide file tree
Showing 2 changed files with 124 additions and 61 deletions.
135 changes: 74 additions & 61 deletions EU3_Scenario_Editor/src/editor/Text.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,10 @@
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.logging.Level;

/**
Expand All @@ -40,15 +42,10 @@ public final class Text {
* @throws IOException
*/
public static void initText(FilenameResolver resolver, GameVersion version) throws FileNotFoundException, IOException {
File[] files = resolver.listFiles("localisation");
if (files == null)
files = resolver.listFiles("localization/english"); // CK3/V3 switched to American spelling and put each language in a separate folder
if (files == null) {
log.log(Level.WARNING, "Could not find localization files");
return;
}
List<File> files = listLocFiles(resolver);

Arrays.sort(files);
if (files.isEmpty())
return;

long startTime = System.currentTimeMillis();
int processedFiles;
Expand All @@ -62,7 +59,29 @@ public static void initText(FilenameResolver resolver, GameVersion version) thro
log.log(Level.INFO, "Processed {0} localization files in {1} ms.", new Object[] { processedFiles, System.currentTimeMillis() - startTime });
}

private static int processFilesCsv(File[] files) throws FileNotFoundException, IOException {
private static List<File> listLocFiles(FilenameResolver resolver) {
// HOI4
List<File> files = resolver.listFilesRecursive("localisation/english");
if (!files.isEmpty()) {
files.addAll(resolver.listFilesRecursive("localisation/replace/english"));
return files;
}

// EU3/EU4
files = resolver.listFilesRecursive("localisation");
if (!files.isEmpty())
return files;

// CK3
files = resolver.listFilesRecursive("localization/english");
if (!files.isEmpty()) {
files.addAll(resolver.listFilesRecursive("localization/replace/english"));
return files;
}
return files;
}

private static int processFilesCsv(List<File> files) throws FileNotFoundException, IOException {
int count = 0;

for (File f : files) {
Expand Down Expand Up @@ -103,78 +122,72 @@ private static int processFilesCsv(File[] files) throws FileNotFoundException, I
return count;
}

private static int processFilesYaml(File[] files) throws FileNotFoundException, IOException {
private static int processFilesYaml(List<File> files) throws FileNotFoundException, IOException {
int count = 0;

// very naive implementation
// EU4 YAML files consist of a single node, defined in the first line
// so we skip that line and break everything else at a ":"
for (File f : files) {
if (f.isDirectory()) {
count += processFilesYaml(f.listFiles());
continue;
}

if (!f.getName().endsWith(".yml"))
continue; // Could use a FileFilter or FilenameFilter

if (f.length() <= 0) {
continue;
}

count++;

int bufferSize = Math.min(102400, (int)f.length());
try (BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(f), StandardCharsets.UTF_8), bufferSize)) {
String line = reader.readLine();

if (line.charAt(0) == '\uFEFF') // Unicode BOM, which Java doesn't handle in UTF-8 files
line = line.substring(1);
if (processYamlFile(f))
count++;
}

return count;
}

if (!line.startsWith("l_english")) // only read English localizations
private static boolean processYamlFile(File f) throws IOException {
int bufferSize = Math.min(102400, (int)f.length());
try (final BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(f), StandardCharsets.UTF_8), bufferSize)) {
// Read the first line and make sure everything is in order
String line = reader.readLine();
if (line.charAt(0) == '\uFEFF') // Unicode BOM, which Java doesn't handle in UTF-8 files
line = line.substring(1);
if (!line.startsWith("l_english")) {
return false;
}

// Read the rest of the file
while ((line = reader.readLine()) != null) {
line = line.trim();
if (line.length() == 0 || line.charAt(0) == '#')
continue;
if (line.charAt(0) == '\uFEFF')
line = line.substring(1);

while ((line = reader.readLine()) != null) {
line = line.trim();
if (line.length() == 0 || line.charAt(0) == '#')
continue;
if (line.charAt(0) == '\uFEFF')
line = line.substring(1);

if (line.endsWith("\\n")) {
StringBuilder lineBuilder = new StringBuilder(line).append("\n").append(line = reader.readLine());
while (line.endsWith("\\n")) {
lineBuilder.append("\n").append(line = reader.readLine());
}
line = lineBuilder.toString();
}

int comment = line.indexOf('#');
if (comment > 0)
line = line.substring(0, comment);

int firstColon = line.indexOf(':');
if (firstColon < 0) {
log.log(Level.WARNING, "Malformed line in file {0}:", f.getPath());
log.log(Level.WARNING, line);
continue;
if (line.endsWith("\\n")) {
StringBuilder lineBuilder = new StringBuilder(line).append("\n").append(line = reader.readLine());
while (line.endsWith("\\n")) {
lineBuilder.append("\n").append(line = reader.readLine());
}

String key = line.substring(0, firstColon).trim(); //.toLowerCase();
//if (!text.containsKey(key)) {
String value = line.substring(firstColon + 1).trim();
value = extractQuote(value);
//if (value.startsWith("\""))
// value = value.substring(1);
//if (value.endsWith("\""))
// value = value.substring(0, value.length() - 1);
text.put(key, value);
//}
line = lineBuilder.toString();
}

int comment = line.indexOf('#');
if (comment > 0)
line = line.substring(0, comment);

int firstColon = line.indexOf(':');
if (firstColon < 0) {
log.log(Level.WARNING, "Malformed line in file {0}:", f.getPath());
log.log(Level.WARNING, line);
continue;
}

String key = line.substring(0, firstColon).trim();
String value = line.substring(firstColon + 1).trim();
value = extractQuote(value);
text.put(key, value);
}
}

return count;
return true;
}

private static String extractQuote(String value) {
Expand Down
50 changes: 50 additions & 0 deletions eugFile/src/eug/shared/FilenameResolver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,56 @@ public File[] listFiles(String dirName) {
}
}


private static String normalizeFileSeparators(String path) {
return path.replace("/", File.separator).replace("\\", File.separator);
}

/**
* Enumerates all files in the given directory and its subfolders. If a mod
* is being used and the directory is set to extend, files in both the
* original and the mod directory are returned (excluding exact path
* duplicates).
* <p>
* Unlike {@link listFiles}, this method is directory-aware:
* that is, if the main folder and the mod folder have a subdirectory
* with the same name, the contents of both will be enumerated. {@code listFiles}
* would not return the subdirectory in the main folder, since it is
* "overridden" by the subdirectory in the mod folder. Therefore, recursively
* calling {@code listFiles} would fail to account for the contents of the
* subdirectory in the main folder.
* @param dirName the name of the directory to list files in.
* @return a list containing the results of {@code File.listFiles()}
* on the given directory in both the main and mod folders.
* @see java.io.File#listFiles()
* @see #listFiles(String)
*/
public java.util.List<File> listFilesRecursive(String dirName) {
// First normalize file separators so we can check substrings later
dirName = normalizeFileSeparators(dirName).toLowerCase();

java.util.List<File> ret = new java.util.ArrayList<>();

File[] files = listFiles(dirName);
if (files == null || files.length == 0)
return ret;

for (File f : files) {
if (f.isFile())
ret.add(f);
else if (f.isDirectory()) {
// grab ONLY the end part and resolve that for recursion
// e.g. turn C:\...\...\Crusader Kings III\game\localization\english\map
// into localization\english\map
String fullPath = f.getAbsolutePath();
String pathEnd = fullPath.substring(fullPath.toLowerCase().indexOf(dirName), fullPath.length());
// now we have only the part that needs to be resolved
ret.addAll(listFilesRecursive(pathEnd));
}
}
return ret;
}

/** Filters out any files with extensions matching {@link #ignoreFileTypes}. */
private File[] filterFiles(File[] files) {
if (files == null || files.length == 0 || ignoreFileTypes.isEmpty())
Expand Down

0 comments on commit a142282

Please sign in to comment.