001////////////////////////////////////////////////////////////////////////////////
002// checkstyle: Checks Java source code for adherence to a set of rules.
003// Copyright (C) 2001-2017 the original author or authors.
004//
005// This library is free software; you can redistribute it and/or
006// modify it under the terms of the GNU Lesser General Public
007// License as published by the Free Software Foundation; either
008// version 2.1 of the License, or (at your option) any later version.
009//
010// This library is distributed in the hope that it will be useful,
011// but WITHOUT ANY WARRANTY; without even the implied warranty of
012// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
013// Lesser General Public License for more details.
014//
015// You should have received a copy of the GNU Lesser General Public
016// License along with this library; if not, write to the Free Software
017// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
018////////////////////////////////////////////////////////////////////////////////
019
020package com.puppycrawl.tools.checkstyle.checks;
021
022import java.io.File;
023import java.io.FileInputStream;
024import java.io.FileNotFoundException;
025import java.io.IOException;
026import java.io.InputStream;
027import java.util.Arrays;
028import java.util.Collections;
029import java.util.HashSet;
030import java.util.Locale;
031import java.util.Optional;
032import java.util.Properties;
033import java.util.Set;
034import java.util.SortedSet;
035import java.util.TreeSet;
036import java.util.regex.Matcher;
037import java.util.regex.Pattern;
038import java.util.stream.Collectors;
039
040import org.apache.commons.logging.Log;
041import org.apache.commons.logging.LogFactory;
042
043import com.google.common.collect.HashMultimap;
044import com.google.common.collect.SetMultimap;
045import com.google.common.io.Closeables;
046import com.puppycrawl.tools.checkstyle.Definitions;
047import com.puppycrawl.tools.checkstyle.api.AbstractFileSetCheck;
048import com.puppycrawl.tools.checkstyle.api.FileText;
049import com.puppycrawl.tools.checkstyle.api.LocalizedMessage;
050import com.puppycrawl.tools.checkstyle.utils.CommonUtils;
051
052/**
053 * <p>
054 * The TranslationCheck class helps to ensure the correct translation of code by
055 * checking locale-specific resource files for consistency regarding their keys.
056 * Two locale-specific resource files describing one and the same context are consistent if they
057 * contain the same keys. TranslationCheck also can check an existence of required translations
058 * which must exist in project, if 'requiredTranslations' option is used.
059 * </p>
060 * <p>
061 * An example of how to configure the check is:
062 * </p>
063 * <pre>
064 * &lt;module name="Translation"/&gt;
065 * </pre>
066 * Check has the following options:
067 *
068 * <p><b>baseName</b> - a base name regexp for resource bundles which contain message resources. It
069 * helps the check to distinguish config and localization resources. Default value is
070 * <b>^messages.*$</b>
071 * <p>An example of how to configure the check to validate only bundles which base names start with
072 * "ButtonLabels":
073 * </p>
074 * <pre>
075 * &lt;module name="Translation"&gt;
076 *     &lt;property name="baseName" value="^ButtonLabels.*$"/&gt;
077 * &lt;/module&gt;
078 * </pre>
079 * <p>To configure the check to check only files which have '.properties' and '.translations'
080 * extensions:
081 * </p>
082 * <pre>
083 * &lt;module name="Translation"&gt;
084 *     &lt;property name="fileExtensions" value="properties, translations"/&gt;
085 * &lt;/module&gt;
086 * </pre>
087 *
088 * <p><b>requiredTranslations</b> which allows to specify language codes of required translations
089 * which must exist in project. Language code is composed of the lowercase, two-letter codes as
090 * defined by <a href="https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes">ISO 639-1</a>.
091 * Default value is <b>empty String Set</b> which means that only the existence of
092 * default translation is checked. Note, if you specify language codes (or just one language
093 * code) of required translations the check will also check for existence of default translation
094 * files in project. ATTENTION: the check will perform the validation of ISO codes if the option
095 * is used. So, if you specify, for example, "mm" for language code, TranslationCheck will rise
096 * violation that the language code is incorrect.
097 * <br>
098 *
099 * @author Alexandra Bunge
100 * @author lkuehne
101 * @author Andrei Selkin
102 */
103public class TranslationCheck extends AbstractFileSetCheck {
104
105    /**
106     * A key is pointing to the warning message text for missing key
107     * in "messages.properties" file.
108     */
109    public static final String MSG_KEY = "translation.missingKey";
110
111    /**
112     * A key is pointing to the warning message text for missing translation file
113     * in "messages.properties" file.
114     */
115    public static final String MSG_KEY_MISSING_TRANSLATION_FILE =
116        "translation.missingTranslationFile";
117
118    /** Resource bundle which contains messages for TranslationCheck. */
119    private static final String TRANSLATION_BUNDLE =
120        "com.puppycrawl.tools.checkstyle.checks.messages";
121
122    /**
123     * A key is pointing to the warning message text for wrong language code
124     * in "messages.properties" file.
125     */
126    private static final String WRONG_LANGUAGE_CODE_KEY = "translation.wrongLanguageCode";
127
128    /**
129     * Regexp string for default translation files.
130     * For example, messages.properties.
131     */
132    private static final String DEFAULT_TRANSLATION_REGEXP = "^.+\\..+$";
133
134    /**
135     * Regexp pattern for bundles names wich end with language code, followed by country code and
136     * variant suffix. For example, messages_es_ES_UNIX.properties.
137     */
138    private static final Pattern LANGUAGE_COUNTRY_VARIANT_PATTERN =
139        CommonUtils.createPattern("^.+\\_[a-z]{2}\\_[A-Z]{2}\\_[A-Za-z]+\\..+$");
140    /**
141     * Regexp pattern for bundles names wich end with language code, followed by country code
142     * suffix. For example, messages_es_ES.properties.
143     */
144    private static final Pattern LANGUAGE_COUNTRY_PATTERN =
145        CommonUtils.createPattern("^.+\\_[a-z]{2}\\_[A-Z]{2}\\..+$");
146    /**
147     * Regexp pattern for bundles names wich end with language code suffix.
148     * For example, messages_es.properties.
149     */
150    private static final Pattern LANGUAGE_PATTERN =
151        CommonUtils.createPattern("^.+\\_[a-z]{2}\\..+$");
152
153    /** File name format for default translation. */
154    private static final String DEFAULT_TRANSLATION_FILE_NAME_FORMATTER = "%s.%s";
155    /** File name format with language code. */
156    private static final String FILE_NAME_WITH_LANGUAGE_CODE_FORMATTER = "%s_%s.%s";
157
158    /** Formatting string to form regexp to validate required translations file names. */
159    private static final String REGEXP_FORMAT_TO_CHECK_REQUIRED_TRANSLATIONS =
160        "^%1$s\\_%2$s(\\_[A-Z]{2})?\\.%3$s$|^%1$s\\_%2$s\\_[A-Z]{2}\\_[A-Za-z]+\\.%3$s$";
161    /** Formatting string to form regexp to validate default translations file names. */
162    private static final String REGEXP_FORMAT_TO_CHECK_DEFAULT_TRANSLATIONS = "^%s\\.%s$";
163
164    /** Logger for TranslationCheck. */
165    private final Log log;
166
167    /** The files to process. */
168    private final Set<File> filesToProcess = new HashSet<>();
169
170    /** The base name regexp pattern. */
171    private Pattern baseName;
172
173    /**
174     * Language codes of required translations for the check (de, pt, ja, etc).
175     */
176    private Set<String> requiredTranslations = new HashSet<>();
177
178    /**
179     * Creates a new {@code TranslationCheck} instance.
180     */
181    public TranslationCheck() {
182        setFileExtensions("properties");
183        baseName = CommonUtils.createPattern("^messages.*$");
184        log = LogFactory.getLog(TranslationCheck.class);
185    }
186
187    /**
188     * Sets the base name regexp pattern.
189     * @param baseName base name regexp.
190     */
191    public void setBaseName(Pattern baseName) {
192        this.baseName = baseName;
193    }
194
195    /**
196     * Sets language codes of required translations for the check.
197     * @param translationCodes a comma separated list of language codes.
198     */
199    public void setRequiredTranslations(String... translationCodes) {
200        requiredTranslations = Arrays.stream(translationCodes).collect(Collectors.toSet());
201        validateUserSpecifiedLanguageCodes(requiredTranslations);
202    }
203
204    /**
205     * Validates the correctness of user specified language codes for the check.
206     * @param languageCodes user specified language codes for the check.
207     */
208    private void validateUserSpecifiedLanguageCodes(Set<String> languageCodes) {
209        for (String code : languageCodes) {
210            if (!isValidLanguageCode(code)) {
211                final LocalizedMessage msg = new LocalizedMessage(0, TRANSLATION_BUNDLE,
212                        WRONG_LANGUAGE_CODE_KEY, new Object[] {code}, getId(), getClass(), null);
213                final String exceptionMessage = String.format(Locale.ROOT,
214                        "%s [%s]", msg.getMessage(), TranslationCheck.class.getSimpleName());
215                throw new IllegalArgumentException(exceptionMessage);
216            }
217        }
218    }
219
220    /**
221     * Checks whether user specified language code is correct (is contained in available locales).
222     * @param userSpecifiedLanguageCode user specified language code.
223     * @return true if user specified language code is correct.
224     */
225    private static boolean isValidLanguageCode(final String userSpecifiedLanguageCode) {
226        boolean valid = false;
227        final Locale[] locales = Locale.getAvailableLocales();
228        for (Locale locale : locales) {
229            if (userSpecifiedLanguageCode.equals(locale.toString())) {
230                valid = true;
231                break;
232            }
233        }
234        return valid;
235    }
236
237    @Override
238    public void beginProcessing(String charset) {
239        filesToProcess.clear();
240    }
241
242    @Override
243    protected void processFiltered(File file, FileText fileText) {
244        // We just collecting files for processing at finishProcessing()
245        filesToProcess.add(file);
246    }
247
248    @Override
249    public void finishProcessing() {
250        final Set<ResourceBundle> bundles = groupFilesIntoBundles(filesToProcess, baseName);
251        for (ResourceBundle currentBundle : bundles) {
252            checkExistenceOfDefaultTranslation(currentBundle);
253            checkExistenceOfRequiredTranslations(currentBundle);
254            checkTranslationKeys(currentBundle);
255        }
256    }
257
258    /**
259     * Checks an existence of default translation file in the resource bundle.
260     * @param bundle resource bundle.
261     */
262    private void checkExistenceOfDefaultTranslation(ResourceBundle bundle) {
263        final Optional<String> fileName = getMissingFileName(bundle, null);
264        if (fileName.isPresent()) {
265            logMissingTranslation(bundle.getPath(), fileName.get());
266        }
267    }
268
269    /**
270     * Checks an existence of translation files in the resource bundle.
271     * The name of translation file begins with the base name of resource bundle which is followed
272     * by '_' and a language code (country and variant are optional), it ends with the extension
273     * suffix.
274     * @param bundle resource bundle.
275     */
276    private void checkExistenceOfRequiredTranslations(ResourceBundle bundle) {
277        for (String languageCode : requiredTranslations) {
278            final Optional<String> fileName = getMissingFileName(bundle, languageCode);
279            if (fileName.isPresent()) {
280                logMissingTranslation(bundle.getPath(), fileName.get());
281            }
282        }
283    }
284
285    /**
286     * Returns the name of translation file which is absent in resource bundle or Guava's Optional,
287     * if there is not missing translation.
288     * @param bundle resource bundle.
289     * @param languageCode language code.
290     * @return the name of translation file which is absent in resource bundle or Guava's Optional,
291     *         if there is not missing translation.
292     */
293    private static Optional<String> getMissingFileName(ResourceBundle bundle, String languageCode) {
294        final String fileNameRegexp;
295        final boolean searchForDefaultTranslation;
296        final String extension = bundle.getExtension();
297        final String baseName = bundle.getBaseName();
298        if (languageCode == null) {
299            searchForDefaultTranslation = true;
300            fileNameRegexp = String.format(Locale.ROOT, REGEXP_FORMAT_TO_CHECK_DEFAULT_TRANSLATIONS,
301                    baseName, extension);
302        }
303        else {
304            searchForDefaultTranslation = false;
305            fileNameRegexp = String.format(Locale.ROOT,
306                REGEXP_FORMAT_TO_CHECK_REQUIRED_TRANSLATIONS, baseName, languageCode, extension);
307        }
308        Optional<String> missingFileName = Optional.empty();
309        if (!bundle.containsFile(fileNameRegexp)) {
310            if (searchForDefaultTranslation) {
311                missingFileName = Optional.of(String.format(Locale.ROOT,
312                        DEFAULT_TRANSLATION_FILE_NAME_FORMATTER, baseName, extension));
313            }
314            else {
315                missingFileName = Optional.of(String.format(Locale.ROOT,
316                        FILE_NAME_WITH_LANGUAGE_CODE_FORMATTER, baseName, languageCode, extension));
317            }
318        }
319        return missingFileName;
320    }
321
322    /**
323     * Logs that translation file is missing.
324     * @param filePath file path.
325     * @param fileName file name.
326     */
327    private void logMissingTranslation(String filePath, String fileName) {
328        log(0, MSG_KEY_MISSING_TRANSLATION_FILE, fileName);
329        fireErrors(filePath);
330        getMessageDispatcher().fireFileFinished(filePath);
331    }
332
333    /**
334     * Groups a set of files into bundles.
335     * Only files, which names match base name regexp pattern will be grouped.
336     * @param files set of files.
337     * @param baseNameRegexp base name regexp pattern.
338     * @return set of ResourceBundles.
339     */
340    private static Set<ResourceBundle> groupFilesIntoBundles(Set<File> files,
341                                                             Pattern baseNameRegexp) {
342        final Set<ResourceBundle> resourceBundles = new HashSet<>();
343        for (File currentFile : files) {
344            final String fileName = currentFile.getName();
345            final String baseName = extractBaseName(fileName);
346            final Matcher baseNameMatcher = baseNameRegexp.matcher(baseName);
347            if (baseNameMatcher.matches()) {
348                final String extension = CommonUtils.getFileExtension(fileName);
349                final String path = getPath(currentFile.getAbsolutePath());
350                final ResourceBundle newBundle = new ResourceBundle(baseName, path, extension);
351                final Optional<ResourceBundle> bundle = findBundle(resourceBundles, newBundle);
352                if (bundle.isPresent()) {
353                    bundle.get().addFile(currentFile);
354                }
355                else {
356                    newBundle.addFile(currentFile);
357                    resourceBundles.add(newBundle);
358                }
359            }
360        }
361        return resourceBundles;
362    }
363
364    /**
365     * Searches for specific resource bundle in a set of resource bundles.
366     * @param bundles set of resource bundles.
367     * @param targetBundle target bundle to search for.
368     * @return Guava's Optional of resource bundle (present if target bundle is found).
369     */
370    private static Optional<ResourceBundle> findBundle(Set<ResourceBundle> bundles,
371                                                       ResourceBundle targetBundle) {
372        Optional<ResourceBundle> result = Optional.empty();
373        for (ResourceBundle currentBundle : bundles) {
374            if (targetBundle.getBaseName().equals(currentBundle.getBaseName())
375                    && targetBundle.getExtension().equals(currentBundle.getExtension())
376                    && targetBundle.getPath().equals(currentBundle.getPath())) {
377                result = Optional.of(currentBundle);
378                break;
379            }
380        }
381        return result;
382    }
383
384    /**
385     * Extracts the base name (the unique prefix) of resource bundle from translation file name.
386     * For example "messages" is the base name of "messages.properties",
387     * "messages_de_AT.properties", "messages_en.properties", etc.
388     * @param fileName the fully qualified name of the translation file.
389     * @return the extracted base name.
390     */
391    private static String extractBaseName(String fileName) {
392        final String regexp;
393        final Matcher languageCountryVariantMatcher =
394            LANGUAGE_COUNTRY_VARIANT_PATTERN.matcher(fileName);
395        final Matcher languageCountryMatcher = LANGUAGE_COUNTRY_PATTERN.matcher(fileName);
396        final Matcher languageMatcher = LANGUAGE_PATTERN.matcher(fileName);
397        if (languageCountryVariantMatcher.matches()) {
398            regexp = LANGUAGE_COUNTRY_VARIANT_PATTERN.pattern();
399        }
400        else if (languageCountryMatcher.matches()) {
401            regexp = LANGUAGE_COUNTRY_PATTERN.pattern();
402        }
403        else if (languageMatcher.matches()) {
404            regexp = LANGUAGE_PATTERN.pattern();
405        }
406        else {
407            regexp = DEFAULT_TRANSLATION_REGEXP;
408        }
409        // We use substring(...) instead of replace(...), so that the regular expression does
410        // not have to be compiled each time it is used inside 'replace' method.
411        final String removePattern = regexp.substring("^.+".length(), regexp.length());
412        return fileName.replaceAll(removePattern, "");
413    }
414
415    /**
416     * Extracts path from a file name which contains the path.
417     * For example, if file nam is /xyz/messages.properties, then the method
418     * will return /xyz/.
419     * @param fileNameWithPath file name which contains the path.
420     * @return file path.
421     */
422    private static String getPath(String fileNameWithPath) {
423        return fileNameWithPath
424            .substring(0, fileNameWithPath.lastIndexOf(File.separator));
425    }
426
427    /**
428     * Checks resource files in bundle for consistency regarding their keys.
429     * All files in bundle must have the same key set. If this is not the case
430     * an error message is posted giving information which key misses in which file.
431     * @param bundle resource bundle.
432     */
433    private void checkTranslationKeys(ResourceBundle bundle) {
434        final Set<File> filesInBundle = bundle.getFiles();
435        if (filesInBundle.size() >= 2) {
436            // build a map from files to the keys they contain
437            final Set<String> allTranslationKeys = new HashSet<>();
438            final SetMultimap<File, String> filesAssociatedWithKeys = HashMultimap.create();
439            for (File currentFile : filesInBundle) {
440                final Set<String> keysInCurrentFile = getTranslationKeys(currentFile);
441                allTranslationKeys.addAll(keysInCurrentFile);
442                filesAssociatedWithKeys.putAll(currentFile, keysInCurrentFile);
443            }
444            checkFilesForConsistencyRegardingTheirKeys(filesAssociatedWithKeys, allTranslationKeys);
445        }
446    }
447
448    /**
449     * Compares th the specified key set with the key sets of the given translation files (arranged
450     * in a map). All missing keys are reported.
451     * @param fileKeys a Map from translation files to their key sets.
452     * @param keysThatMustExist the set of keys to compare with.
453     */
454    private void checkFilesForConsistencyRegardingTheirKeys(SetMultimap<File, String> fileKeys,
455                                                            Set<String> keysThatMustExist) {
456        for (File currentFile : fileKeys.keySet()) {
457            final Set<String> currentFileKeys = fileKeys.get(currentFile);
458            final Set<String> missingKeys = keysThatMustExist.stream()
459                .filter(e -> !currentFileKeys.contains(e)).collect(Collectors.toSet());
460            if (!missingKeys.isEmpty()) {
461                for (Object key : missingKeys) {
462                    log(0, MSG_KEY, key);
463                }
464            }
465            final String path = currentFile.getPath();
466            fireErrors(path);
467            getMessageDispatcher().fireFileFinished(path);
468        }
469    }
470
471    /**
472     * Loads the keys from the specified translation file into a set.
473     * @param file translation file.
474     * @return a Set object which holds the loaded keys.
475     */
476    private Set<String> getTranslationKeys(File file) {
477        Set<String> keys = new HashSet<>();
478        InputStream inStream = null;
479        try {
480            inStream = new FileInputStream(file);
481            final Properties translations = new Properties();
482            translations.load(inStream);
483            keys = translations.stringPropertyNames();
484        }
485        catch (final IOException ex) {
486            logIoException(ex, file);
487        }
488        finally {
489            Closeables.closeQuietly(inStream);
490        }
491        return keys;
492    }
493
494    /**
495     * Helper method to log an io exception.
496     * @param exception the exception that occurred
497     * @param file the file that could not be processed
498     */
499    private void logIoException(IOException exception, File file) {
500        String[] args = null;
501        String key = "general.fileNotFound";
502        if (!(exception instanceof FileNotFoundException)) {
503            args = new String[] {exception.getMessage()};
504            key = "general.exception";
505        }
506        final LocalizedMessage message =
507            new LocalizedMessage(
508                0,
509                Definitions.CHECKSTYLE_BUNDLE,
510                key,
511                args,
512                getId(),
513                getClass(), null);
514        final SortedSet<LocalizedMessage> messages = new TreeSet<>();
515        messages.add(message);
516        getMessageDispatcher().fireErrors(file.getPath(), messages);
517        log.debug("IOException occurred.", exception);
518    }
519
520    /** Class which represents a resource bundle. */
521    private static class ResourceBundle {
522        /** Bundle base name. */
523        private final String baseName;
524        /** Common extension of files which are included in the resource bundle. */
525        private final String extension;
526        /** Common path of files which are included in the resource bundle. */
527        private final String path;
528        /** Set of files which are included in the resource bundle. */
529        private final Set<File> files;
530
531        /**
532         * Creates a ResourceBundle object with specific base name, common files extension.
533         * @param baseName bundle base name.
534         * @param path common path of files which are included in the resource bundle.
535         * @param extension common extension of files which are included in the resource bundle.
536         */
537        ResourceBundle(String baseName, String path, String extension) {
538            this.baseName = baseName;
539            this.path = path;
540            this.extension = extension;
541            files = new HashSet<>();
542        }
543
544        public String getBaseName() {
545            return baseName;
546        }
547
548        public String getPath() {
549            return path;
550        }
551
552        public String getExtension() {
553            return extension;
554        }
555
556        public Set<File> getFiles() {
557            return Collections.unmodifiableSet(files);
558        }
559
560        /**
561         * Adds a file into resource bundle.
562         * @param file file which should be added into resource bundle.
563         */
564        public void addFile(File file) {
565            files.add(file);
566        }
567
568        /**
569         * Checks whether a resource bundle contains a file which name matches file name regexp.
570         * @param fileNameRegexp file name regexp.
571         * @return true if a resource bundle contains a file which name matches file name regexp.
572         */
573        public boolean containsFile(String fileNameRegexp) {
574            boolean containsFile = false;
575            for (File currentFile : files) {
576                if (Pattern.matches(fileNameRegexp, currentFile.getName())) {
577                    containsFile = true;
578                    break;
579                }
580            }
581            return containsFile;
582        }
583    }
584}