001////////////////////////////////////////////////////////////////////////////////
002// checkstyle: Checks Java source code for adherence to a set of rules.
003// Copyright (C) 2001-2018 the original author or authors.
004//
005// This library is free software; you can redistribute it and/or
006// modify it under the terms of the GNU Lesser General Public
007// License as published by the Free Software Foundation; either
008// version 2.1 of the License, or (at your option) any later version.
009//
010// This library is distributed in the hope that it will be useful,
011// but WITHOUT ANY WARRANTY; without even the implied warranty of
012// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
013// Lesser General Public License for more details.
014//
015// You should have received a copy of the GNU Lesser General Public
016// License along with this library; if not, write to the Free Software
017// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
018////////////////////////////////////////////////////////////////////////////////
019
020package com.puppycrawl.tools.checkstyle.checks;
021
022import java.io.File;
023import java.io.IOException;
024import java.io.InputStream;
025import java.nio.file.Files;
026import java.nio.file.NoSuchFileException;
027import java.util.Arrays;
028import java.util.Collections;
029import java.util.HashSet;
030import java.util.Locale;
031import java.util.Optional;
032import java.util.Properties;
033import java.util.Set;
034import java.util.SortedSet;
035import java.util.TreeSet;
036import java.util.regex.Matcher;
037import java.util.regex.Pattern;
038import java.util.stream.Collectors;
039
040import org.apache.commons.logging.Log;
041import org.apache.commons.logging.LogFactory;
042
043import com.google.common.collect.HashMultimap;
044import com.google.common.collect.SetMultimap;
045import com.puppycrawl.tools.checkstyle.Definitions;
046import com.puppycrawl.tools.checkstyle.api.AbstractFileSetCheck;
047import com.puppycrawl.tools.checkstyle.api.FileText;
048import com.puppycrawl.tools.checkstyle.api.LocalizedMessage;
049import com.puppycrawl.tools.checkstyle.api.MessageDispatcher;
050import com.puppycrawl.tools.checkstyle.utils.CommonUtils;
051
052/**
053 * <p>
054 * The TranslationCheck class helps to ensure the correct translation of code by
055 * checking locale-specific resource files for consistency regarding their keys.
056 * Two locale-specific resource files describing one and the same context are consistent if they
057 * contain the same keys. TranslationCheck also can check an existence of required translations
058 * which must exist in project, if 'requiredTranslations' option is used.
059 * </p>
060 * <p>
061 * An example of how to configure the check is:
062 * </p>
063 * <pre>
064 * &lt;module name="Translation"/&gt;
065 * </pre>
066 * Check has the following options:
067 *
068 * <p><b>baseName</b> - a base name regexp for resource bundles which contain message resources. It
069 * helps the check to distinguish config and localization resources. Default value is
070 * <b>^messages.*$</b>
071 * <p>An example of how to configure the check to validate only bundles which base names start with
072 * "ButtonLabels":
073 * </p>
074 * <pre>
075 * &lt;module name="Translation"&gt;
076 *     &lt;property name="baseName" value="^ButtonLabels.*$"/&gt;
077 * &lt;/module&gt;
078 * </pre>
079 * <p>To configure the check to check only files which have '.properties' and '.translations'
080 * extensions:
081 * </p>
082 * <pre>
083 * &lt;module name="Translation"&gt;
084 *     &lt;property name="fileExtensions" value="properties, translations"/&gt;
085 * &lt;/module&gt;
086 * </pre>
087 *
088 * <p><b>requiredTranslations</b> which allows to specify language codes of required translations
089 * which must exist in project. Language code is composed of the lowercase, two-letter codes as
090 * defined by <a href="https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes">ISO 639-1</a>.
091 * Default value is <b>empty String Set</b> which means that only the existence of
092 * default translation is checked. Note, if you specify language codes (or just one language
093 * code) of required translations the check will also check for existence of default translation
094 * files in project. ATTENTION: the check will perform the validation of ISO codes if the option
095 * is used. So, if you specify, for example, "mm" for language code, TranslationCheck will rise
096 * violation that the language code is incorrect.
097 * <br>
098 *
099 */
100public class TranslationCheck extends AbstractFileSetCheck {
101
102    /**
103     * A key is pointing to the warning message text for missing key
104     * in "messages.properties" file.
105     */
106    public static final String MSG_KEY = "translation.missingKey";
107
108    /**
109     * A key is pointing to the warning message text for missing translation file
110     * in "messages.properties" file.
111     */
112    public static final String MSG_KEY_MISSING_TRANSLATION_FILE =
113        "translation.missingTranslationFile";
114
115    /** Resource bundle which contains messages for TranslationCheck. */
116    private static final String TRANSLATION_BUNDLE =
117        "com.puppycrawl.tools.checkstyle.checks.messages";
118
119    /**
120     * A key is pointing to the warning message text for wrong language code
121     * in "messages.properties" file.
122     */
123    private static final String WRONG_LANGUAGE_CODE_KEY = "translation.wrongLanguageCode";
124
125    /**
126     * Regexp string for default translation files.
127     * For example, messages.properties.
128     */
129    private static final String DEFAULT_TRANSLATION_REGEXP = "^.+\\..+$";
130
131    /**
132     * Regexp pattern for bundles names which end with language code, followed by country code and
133     * variant suffix. For example, messages_es_ES_UNIX.properties.
134     */
135    private static final Pattern LANGUAGE_COUNTRY_VARIANT_PATTERN =
136        CommonUtils.createPattern("^.+\\_[a-z]{2}\\_[A-Z]{2}\\_[A-Za-z]+\\..+$");
137    /**
138     * Regexp pattern for bundles names which end with language code, followed by country code
139     * suffix. For example, messages_es_ES.properties.
140     */
141    private static final Pattern LANGUAGE_COUNTRY_PATTERN =
142        CommonUtils.createPattern("^.+\\_[a-z]{2}\\_[A-Z]{2}\\..+$");
143    /**
144     * Regexp pattern for bundles names which end with language code suffix.
145     * For example, messages_es.properties.
146     */
147    private static final Pattern LANGUAGE_PATTERN =
148        CommonUtils.createPattern("^.+\\_[a-z]{2}\\..+$");
149
150    /** File name format for default translation. */
151    private static final String DEFAULT_TRANSLATION_FILE_NAME_FORMATTER = "%s.%s";
152    /** File name format with language code. */
153    private static final String FILE_NAME_WITH_LANGUAGE_CODE_FORMATTER = "%s_%s.%s";
154
155    /** Formatting string to form regexp to validate required translations file names. */
156    private static final String REGEXP_FORMAT_TO_CHECK_REQUIRED_TRANSLATIONS =
157        "^%1$s\\_%2$s(\\_[A-Z]{2})?\\.%3$s$|^%1$s\\_%2$s\\_[A-Z]{2}\\_[A-Za-z]+\\.%3$s$";
158    /** Formatting string to form regexp to validate default translations file names. */
159    private static final String REGEXP_FORMAT_TO_CHECK_DEFAULT_TRANSLATIONS = "^%s\\.%s$";
160
161    /** Logger for TranslationCheck. */
162    private final Log log;
163
164    /** The files to process. */
165    private final Set<File> filesToProcess = new HashSet<>();
166
167    /** The base name regexp pattern. */
168    private Pattern baseName;
169
170    /**
171     * Language codes of required translations for the check (de, pt, ja, etc).
172     */
173    private Set<String> requiredTranslations = new HashSet<>();
174
175    /**
176     * Creates a new {@code TranslationCheck} instance.
177     */
178    public TranslationCheck() {
179        setFileExtensions("properties");
180        baseName = CommonUtils.createPattern("^messages.*$");
181        log = LogFactory.getLog(TranslationCheck.class);
182    }
183
184    /**
185     * Sets the base name regexp pattern.
186     * @param baseName base name regexp.
187     */
188    public void setBaseName(Pattern baseName) {
189        this.baseName = baseName;
190    }
191
192    /**
193     * Sets language codes of required translations for the check.
194     * @param translationCodes a comma separated list of language codes.
195     */
196    public void setRequiredTranslations(String... translationCodes) {
197        requiredTranslations = Arrays.stream(translationCodes).collect(Collectors.toSet());
198        validateUserSpecifiedLanguageCodes(requiredTranslations);
199    }
200
201    /**
202     * Validates the correctness of user specified language codes for the check.
203     * @param languageCodes user specified language codes for the check.
204     */
205    private void validateUserSpecifiedLanguageCodes(Set<String> languageCodes) {
206        for (String code : languageCodes) {
207            if (!isValidLanguageCode(code)) {
208                final LocalizedMessage msg = new LocalizedMessage(0, TRANSLATION_BUNDLE,
209                        WRONG_LANGUAGE_CODE_KEY, new Object[] {code}, getId(), getClass(), null);
210                final String exceptionMessage = String.format(Locale.ROOT,
211                        "%s [%s]", msg.getMessage(), TranslationCheck.class.getSimpleName());
212                throw new IllegalArgumentException(exceptionMessage);
213            }
214        }
215    }
216
217    /**
218     * Checks whether user specified language code is correct (is contained in available locales).
219     * @param userSpecifiedLanguageCode user specified language code.
220     * @return true if user specified language code is correct.
221     */
222    private static boolean isValidLanguageCode(final String userSpecifiedLanguageCode) {
223        boolean valid = false;
224        final Locale[] locales = Locale.getAvailableLocales();
225        for (Locale locale : locales) {
226            if (userSpecifiedLanguageCode.equals(locale.toString())) {
227                valid = true;
228                break;
229            }
230        }
231        return valid;
232    }
233
234    @Override
235    public void beginProcessing(String charset) {
236        filesToProcess.clear();
237    }
238
239    @Override
240    protected void processFiltered(File file, FileText fileText) {
241        // We just collecting files for processing at finishProcessing()
242        filesToProcess.add(file);
243    }
244
245    @Override
246    public void finishProcessing() {
247        final Set<ResourceBundle> bundles = groupFilesIntoBundles(filesToProcess, baseName);
248        for (ResourceBundle currentBundle : bundles) {
249            checkExistenceOfDefaultTranslation(currentBundle);
250            checkExistenceOfRequiredTranslations(currentBundle);
251            checkTranslationKeys(currentBundle);
252        }
253    }
254
255    /**
256     * Checks an existence of default translation file in the resource bundle.
257     * @param bundle resource bundle.
258     */
259    private void checkExistenceOfDefaultTranslation(ResourceBundle bundle) {
260        final Optional<String> fileName = getMissingFileName(bundle, null);
261        if (fileName.isPresent()) {
262            logMissingTranslation(bundle.getPath(), fileName.get());
263        }
264    }
265
266    /**
267     * Checks an existence of translation files in the resource bundle.
268     * The name of translation file begins with the base name of resource bundle which is followed
269     * by '_' and a language code (country and variant are optional), it ends with the extension
270     * suffix.
271     * @param bundle resource bundle.
272     */
273    private void checkExistenceOfRequiredTranslations(ResourceBundle bundle) {
274        for (String languageCode : requiredTranslations) {
275            final Optional<String> fileName = getMissingFileName(bundle, languageCode);
276            if (fileName.isPresent()) {
277                logMissingTranslation(bundle.getPath(), fileName.get());
278            }
279        }
280    }
281
282    /**
283     * Returns the name of translation file which is absent in resource bundle or Guava's Optional,
284     * if there is not missing translation.
285     * @param bundle resource bundle.
286     * @param languageCode language code.
287     * @return the name of translation file which is absent in resource bundle or Guava's Optional,
288     *         if there is not missing translation.
289     */
290    private static Optional<String> getMissingFileName(ResourceBundle bundle, String languageCode) {
291        final String fileNameRegexp;
292        final boolean searchForDefaultTranslation;
293        final String extension = bundle.getExtension();
294        final String baseName = bundle.getBaseName();
295        if (languageCode == null) {
296            searchForDefaultTranslation = true;
297            fileNameRegexp = String.format(Locale.ROOT, REGEXP_FORMAT_TO_CHECK_DEFAULT_TRANSLATIONS,
298                    baseName, extension);
299        }
300        else {
301            searchForDefaultTranslation = false;
302            fileNameRegexp = String.format(Locale.ROOT,
303                REGEXP_FORMAT_TO_CHECK_REQUIRED_TRANSLATIONS, baseName, languageCode, extension);
304        }
305        Optional<String> missingFileName = Optional.empty();
306        if (!bundle.containsFile(fileNameRegexp)) {
307            if (searchForDefaultTranslation) {
308                missingFileName = Optional.of(String.format(Locale.ROOT,
309                        DEFAULT_TRANSLATION_FILE_NAME_FORMATTER, baseName, extension));
310            }
311            else {
312                missingFileName = Optional.of(String.format(Locale.ROOT,
313                        FILE_NAME_WITH_LANGUAGE_CODE_FORMATTER, baseName, languageCode, extension));
314            }
315        }
316        return missingFileName;
317    }
318
319    /**
320     * Logs that translation file is missing.
321     * @param filePath file path.
322     * @param fileName file name.
323     */
324    private void logMissingTranslation(String filePath, String fileName) {
325        final MessageDispatcher dispatcher = getMessageDispatcher();
326        dispatcher.fireFileStarted(filePath);
327        log(0, MSG_KEY_MISSING_TRANSLATION_FILE, fileName);
328        fireErrors(filePath);
329        dispatcher.fireFileFinished(filePath);
330    }
331
332    /**
333     * Groups a set of files into bundles.
334     * Only files, which names match base name regexp pattern will be grouped.
335     * @param files set of files.
336     * @param baseNameRegexp base name regexp pattern.
337     * @return set of ResourceBundles.
338     */
339    private static Set<ResourceBundle> groupFilesIntoBundles(Set<File> files,
340                                                             Pattern baseNameRegexp) {
341        final Set<ResourceBundle> resourceBundles = new HashSet<>();
342        for (File currentFile : files) {
343            final String fileName = currentFile.getName();
344            final String baseName = extractBaseName(fileName);
345            final Matcher baseNameMatcher = baseNameRegexp.matcher(baseName);
346            if (baseNameMatcher.matches()) {
347                final String extension = CommonUtils.getFileExtension(fileName);
348                final String path = getPath(currentFile.getAbsolutePath());
349                final ResourceBundle newBundle = new ResourceBundle(baseName, path, extension);
350                final Optional<ResourceBundle> bundle = findBundle(resourceBundles, newBundle);
351                if (bundle.isPresent()) {
352                    bundle.get().addFile(currentFile);
353                }
354                else {
355                    newBundle.addFile(currentFile);
356                    resourceBundles.add(newBundle);
357                }
358            }
359        }
360        return resourceBundles;
361    }
362
363    /**
364     * Searches for specific resource bundle in a set of resource bundles.
365     * @param bundles set of resource bundles.
366     * @param targetBundle target bundle to search for.
367     * @return Guava's Optional of resource bundle (present if target bundle is found).
368     */
369    private static Optional<ResourceBundle> findBundle(Set<ResourceBundle> bundles,
370                                                       ResourceBundle targetBundle) {
371        Optional<ResourceBundle> result = Optional.empty();
372        for (ResourceBundle currentBundle : bundles) {
373            if (targetBundle.getBaseName().equals(currentBundle.getBaseName())
374                    && targetBundle.getExtension().equals(currentBundle.getExtension())
375                    && targetBundle.getPath().equals(currentBundle.getPath())) {
376                result = Optional.of(currentBundle);
377                break;
378            }
379        }
380        return result;
381    }
382
383    /**
384     * Extracts the base name (the unique prefix) of resource bundle from translation file name.
385     * For example "messages" is the base name of "messages.properties",
386     * "messages_de_AT.properties", "messages_en.properties", etc.
387     * @param fileName the fully qualified name of the translation file.
388     * @return the extracted base name.
389     */
390    private static String extractBaseName(String fileName) {
391        final String regexp;
392        final Matcher languageCountryVariantMatcher =
393            LANGUAGE_COUNTRY_VARIANT_PATTERN.matcher(fileName);
394        final Matcher languageCountryMatcher = LANGUAGE_COUNTRY_PATTERN.matcher(fileName);
395        final Matcher languageMatcher = LANGUAGE_PATTERN.matcher(fileName);
396        if (languageCountryVariantMatcher.matches()) {
397            regexp = LANGUAGE_COUNTRY_VARIANT_PATTERN.pattern();
398        }
399        else if (languageCountryMatcher.matches()) {
400            regexp = LANGUAGE_COUNTRY_PATTERN.pattern();
401        }
402        else if (languageMatcher.matches()) {
403            regexp = LANGUAGE_PATTERN.pattern();
404        }
405        else {
406            regexp = DEFAULT_TRANSLATION_REGEXP;
407        }
408        // We use substring(...) instead of replace(...), so that the regular expression does
409        // not have to be compiled each time it is used inside 'replace' method.
410        final String removePattern = regexp.substring("^.+".length(), regexp.length());
411        return fileName.replaceAll(removePattern, "");
412    }
413
414    /**
415     * Extracts path from a file name which contains the path.
416     * For example, if file nam is /xyz/messages.properties, then the method
417     * will return /xyz/.
418     * @param fileNameWithPath file name which contains the path.
419     * @return file path.
420     */
421    private static String getPath(String fileNameWithPath) {
422        return fileNameWithPath
423            .substring(0, fileNameWithPath.lastIndexOf(File.separator));
424    }
425
426    /**
427     * Checks resource files in bundle for consistency regarding their keys.
428     * All files in bundle must have the same key set. If this is not the case
429     * an error message is posted giving information which key misses in which file.
430     * @param bundle resource bundle.
431     */
432    private void checkTranslationKeys(ResourceBundle bundle) {
433        final Set<File> filesInBundle = bundle.getFiles();
434        if (filesInBundle.size() >= 2) {
435            // build a map from files to the keys they contain
436            final Set<String> allTranslationKeys = new HashSet<>();
437            final SetMultimap<File, String> filesAssociatedWithKeys = HashMultimap.create();
438            for (File currentFile : filesInBundle) {
439                final Set<String> keysInCurrentFile = getTranslationKeys(currentFile);
440                allTranslationKeys.addAll(keysInCurrentFile);
441                filesAssociatedWithKeys.putAll(currentFile, keysInCurrentFile);
442            }
443            checkFilesForConsistencyRegardingTheirKeys(filesAssociatedWithKeys, allTranslationKeys);
444        }
445    }
446
447    /**
448     * Compares th the specified key set with the key sets of the given translation files (arranged
449     * in a map). All missing keys are reported.
450     * @param fileKeys a Map from translation files to their key sets.
451     * @param keysThatMustExist the set of keys to compare with.
452     */
453    private void checkFilesForConsistencyRegardingTheirKeys(SetMultimap<File, String> fileKeys,
454                                                            Set<String> keysThatMustExist) {
455        for (File currentFile : fileKeys.keySet()) {
456            final MessageDispatcher dispatcher = getMessageDispatcher();
457            final String path = currentFile.getPath();
458            dispatcher.fireFileStarted(path);
459            final Set<String> currentFileKeys = fileKeys.get(currentFile);
460            final Set<String> missingKeys = keysThatMustExist.stream()
461                .filter(e -> !currentFileKeys.contains(e)).collect(Collectors.toSet());
462            if (!missingKeys.isEmpty()) {
463                for (Object key : missingKeys) {
464                    log(0, MSG_KEY, key);
465                }
466            }
467            fireErrors(path);
468            dispatcher.fireFileFinished(path);
469        }
470    }
471
472    /**
473     * Loads the keys from the specified translation file into a set.
474     * @param file translation file.
475     * @return a Set object which holds the loaded keys.
476     */
477    private Set<String> getTranslationKeys(File file) {
478        Set<String> keys = new HashSet<>();
479        try (InputStream inStream = Files.newInputStream(file.toPath())) {
480            final Properties translations = new Properties();
481            translations.load(inStream);
482            keys = translations.stringPropertyNames();
483        }
484        catch (final IOException ex) {
485            logIoException(ex, file);
486        }
487        return keys;
488    }
489
490    /**
491     * Helper method to log an io exception.
492     * @param exception the exception that occurred
493     * @param file the file that could not be processed
494     */
495    private void logIoException(IOException exception, File file) {
496        String[] args = null;
497        String key = "general.fileNotFound";
498        if (!(exception instanceof NoSuchFileException)) {
499            args = new String[] {exception.getMessage()};
500            key = "general.exception";
501        }
502        final LocalizedMessage message =
503            new LocalizedMessage(
504                0,
505                Definitions.CHECKSTYLE_BUNDLE,
506                key,
507                args,
508                getId(),
509                getClass(), null);
510        final SortedSet<LocalizedMessage> messages = new TreeSet<>();
511        messages.add(message);
512        getMessageDispatcher().fireErrors(file.getPath(), messages);
513        log.debug("IOException occurred.", exception);
514    }
515
516    /** Class which represents a resource bundle. */
517    private static class ResourceBundle {
518
519        /** Bundle base name. */
520        private final String baseName;
521        /** Common extension of files which are included in the resource bundle. */
522        private final String extension;
523        /** Common path of files which are included in the resource bundle. */
524        private final String path;
525        /** Set of files which are included in the resource bundle. */
526        private final Set<File> files;
527
528        /**
529         * Creates a ResourceBundle object with specific base name, common files extension.
530         * @param baseName bundle base name.
531         * @param path common path of files which are included in the resource bundle.
532         * @param extension common extension of files which are included in the resource bundle.
533         */
534        ResourceBundle(String baseName, String path, String extension) {
535            this.baseName = baseName;
536            this.path = path;
537            this.extension = extension;
538            files = new HashSet<>();
539        }
540
541        public String getBaseName() {
542            return baseName;
543        }
544
545        public String getPath() {
546            return path;
547        }
548
549        public String getExtension() {
550            return extension;
551        }
552
553        public Set<File> getFiles() {
554            return Collections.unmodifiableSet(files);
555        }
556
557        /**
558         * Adds a file into resource bundle.
559         * @param file file which should be added into resource bundle.
560         */
561        public void addFile(File file) {
562            files.add(file);
563        }
564
565        /**
566         * Checks whether a resource bundle contains a file which name matches file name regexp.
567         * @param fileNameRegexp file name regexp.
568         * @return true if a resource bundle contains a file which name matches file name regexp.
569         */
570        public boolean containsFile(String fileNameRegexp) {
571            boolean containsFile = false;
572            for (File currentFile : files) {
573                if (Pattern.matches(fileNameRegexp, currentFile.getName())) {
574                    containsFile = true;
575                    break;
576                }
577            }
578            return containsFile;
579        }
580
581    }
582
583}