View Javadoc
1   ////////////////////////////////////////////////////////////////////////////////
2   // checkstyle: Checks Java source code for adherence to a set of rules.
3   // Copyright (C) 2001-2017 the original author or authors.
4   //
5   // This library is free software; you can redistribute it and/or
6   // modify it under the terms of the GNU Lesser General Public
7   // License as published by the Free Software Foundation; either
8   // version 2.1 of the License, or (at your option) any later version.
9   //
10  // This library is distributed in the hope that it will be useful,
11  // but WITHOUT ANY WARRANTY; without even the implied warranty of
12  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13  // Lesser General Public License for more details.
14  //
15  // You should have received a copy of the GNU Lesser General Public
16  // License along with this library; if not, write to the Free Software
17  // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
18  ////////////////////////////////////////////////////////////////////////////////
19  
20  package com.puppycrawl.tools.checkstyle.checks;
21  
22  import java.io.File;
23  import java.io.FileInputStream;
24  import java.io.FileNotFoundException;
25  import java.io.IOException;
26  import java.io.InputStream;
27  import java.util.Arrays;
28  import java.util.Collections;
29  import java.util.HashSet;
30  import java.util.Locale;
31  import java.util.Optional;
32  import java.util.Properties;
33  import java.util.Set;
34  import java.util.SortedSet;
35  import java.util.TreeSet;
36  import java.util.regex.Matcher;
37  import java.util.regex.Pattern;
38  import java.util.stream.Collectors;
39  
40  import org.apache.commons.logging.Log;
41  import org.apache.commons.logging.LogFactory;
42  
43  import com.google.common.collect.HashMultimap;
44  import com.google.common.collect.SetMultimap;
45  import com.google.common.io.Closeables;
46  import com.puppycrawl.tools.checkstyle.Definitions;
47  import com.puppycrawl.tools.checkstyle.api.AbstractFileSetCheck;
48  import com.puppycrawl.tools.checkstyle.api.FileText;
49  import com.puppycrawl.tools.checkstyle.api.LocalizedMessage;
50  import com.puppycrawl.tools.checkstyle.api.MessageDispatcher;
51  import com.puppycrawl.tools.checkstyle.utils.CommonUtils;
52  
53  /**
54   * <p>
55   * The TranslationCheck class helps to ensure the correct translation of code by
56   * checking locale-specific resource files for consistency regarding their keys.
57   * Two locale-specific resource files describing one and the same context are consistent if they
58   * contain the same keys. TranslationCheck also can check an existence of required translations
59   * which must exist in project, if 'requiredTranslations' option is used.
60   * </p>
61   * <p>
62   * An example of how to configure the check is:
63   * </p>
64   * <pre>
65   * &lt;module name="Translation"/&gt;
66   * </pre>
67   * Check has the following options:
68   *
69   * <p><b>baseName</b> - a base name regexp for resource bundles which contain message resources. It
70   * helps the check to distinguish config and localization resources. Default value is
71   * <b>^messages.*$</b>
72   * <p>An example of how to configure the check to validate only bundles which base names start with
73   * "ButtonLabels":
74   * </p>
75   * <pre>
76   * &lt;module name="Translation"&gt;
77   *     &lt;property name="baseName" value="^ButtonLabels.*$"/&gt;
78   * &lt;/module&gt;
79   * </pre>
80   * <p>To configure the check to check only files which have '.properties' and '.translations'
81   * extensions:
82   * </p>
83   * <pre>
84   * &lt;module name="Translation"&gt;
85   *     &lt;property name="fileExtensions" value="properties, translations"/&gt;
86   * &lt;/module&gt;
87   * </pre>
88   *
89   * <p><b>requiredTranslations</b> which allows to specify language codes of required translations
90   * which must exist in project. Language code is composed of the lowercase, two-letter codes as
91   * defined by <a href="https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes">ISO 639-1</a>.
92   * Default value is <b>empty String Set</b> which means that only the existence of
93   * default translation is checked. Note, if you specify language codes (or just one language
94   * code) of required translations the check will also check for existence of default translation
95   * files in project. ATTENTION: the check will perform the validation of ISO codes if the option
96   * is used. So, if you specify, for example, "mm" for language code, TranslationCheck will rise
97   * violation that the language code is incorrect.
98   * <br>
99   *
100  * @author Alexandra Bunge
101  * @author lkuehne
102  * @author Andrei Selkin
103  */
104 public class TranslationCheck extends AbstractFileSetCheck {
105 
106     /**
107      * A key is pointing to the warning message text for missing key
108      * in "messages.properties" file.
109      */
110     public static final String MSG_KEY = "translation.missingKey";
111 
112     /**
113      * A key is pointing to the warning message text for missing translation file
114      * in "messages.properties" file.
115      */
116     public static final String MSG_KEY_MISSING_TRANSLATION_FILE =
117         "translation.missingTranslationFile";
118 
119     /** Resource bundle which contains messages for TranslationCheck. */
120     private static final String TRANSLATION_BUNDLE =
121         "com.puppycrawl.tools.checkstyle.checks.messages";
122 
123     /**
124      * A key is pointing to the warning message text for wrong language code
125      * in "messages.properties" file.
126      */
127     private static final String WRONG_LANGUAGE_CODE_KEY = "translation.wrongLanguageCode";
128 
129     /**
130      * Regexp string for default translation files.
131      * For example, messages.properties.
132      */
133     private static final String DEFAULT_TRANSLATION_REGEXP = "^.+\\..+$";
134 
135     /**
136      * Regexp pattern for bundles names which end with language code, followed by country code and
137      * variant suffix. For example, messages_es_ES_UNIX.properties.
138      */
139     private static final Pattern LANGUAGE_COUNTRY_VARIANT_PATTERN =
140         CommonUtils.createPattern("^.+\\_[a-z]{2}\\_[A-Z]{2}\\_[A-Za-z]+\\..+$");
141     /**
142      * Regexp pattern for bundles names which end with language code, followed by country code
143      * suffix. For example, messages_es_ES.properties.
144      */
145     private static final Pattern LANGUAGE_COUNTRY_PATTERN =
146         CommonUtils.createPattern("^.+\\_[a-z]{2}\\_[A-Z]{2}\\..+$");
147     /**
148      * Regexp pattern for bundles names which end with language code suffix.
149      * For example, messages_es.properties.
150      */
151     private static final Pattern LANGUAGE_PATTERN =
152         CommonUtils.createPattern("^.+\\_[a-z]{2}\\..+$");
153 
154     /** File name format for default translation. */
155     private static final String DEFAULT_TRANSLATION_FILE_NAME_FORMATTER = "%s.%s";
156     /** File name format with language code. */
157     private static final String FILE_NAME_WITH_LANGUAGE_CODE_FORMATTER = "%s_%s.%s";
158 
159     /** Formatting string to form regexp to validate required translations file names. */
160     private static final String REGEXP_FORMAT_TO_CHECK_REQUIRED_TRANSLATIONS =
161         "^%1$s\\_%2$s(\\_[A-Z]{2})?\\.%3$s$|^%1$s\\_%2$s\\_[A-Z]{2}\\_[A-Za-z]+\\.%3$s$";
162     /** Formatting string to form regexp to validate default translations file names. */
163     private static final String REGEXP_FORMAT_TO_CHECK_DEFAULT_TRANSLATIONS = "^%s\\.%s$";
164 
165     /** Logger for TranslationCheck. */
166     private final Log log;
167 
168     /** The files to process. */
169     private final Set<File> filesToProcess = new HashSet<>();
170 
171     /** The base name regexp pattern. */
172     private Pattern baseName;
173 
174     /**
175      * Language codes of required translations for the check (de, pt, ja, etc).
176      */
177     private Set<String> requiredTranslations = new HashSet<>();
178 
179     /**
180      * Creates a new {@code TranslationCheck} instance.
181      */
182     public TranslationCheck() {
183         setFileExtensions("properties");
184         baseName = CommonUtils.createPattern("^messages.*$");
185         log = LogFactory.getLog(TranslationCheck.class);
186     }
187 
188     /**
189      * Sets the base name regexp pattern.
190      * @param baseName base name regexp.
191      */
192     public void setBaseName(Pattern baseName) {
193         this.baseName = baseName;
194     }
195 
196     /**
197      * Sets language codes of required translations for the check.
198      * @param translationCodes a comma separated list of language codes.
199      */
200     public void setRequiredTranslations(String... translationCodes) {
201         requiredTranslations = Arrays.stream(translationCodes).collect(Collectors.toSet());
202         validateUserSpecifiedLanguageCodes(requiredTranslations);
203     }
204 
205     /**
206      * Validates the correctness of user specified language codes for the check.
207      * @param languageCodes user specified language codes for the check.
208      */
209     private void validateUserSpecifiedLanguageCodes(Set<String> languageCodes) {
210         for (String code : languageCodes) {
211             if (!isValidLanguageCode(code)) {
212                 final LocalizedMessage msg = new LocalizedMessage(0, TRANSLATION_BUNDLE,
213                         WRONG_LANGUAGE_CODE_KEY, new Object[] {code}, getId(), getClass(), null);
214                 final String exceptionMessage = String.format(Locale.ROOT,
215                         "%s [%s]", msg.getMessage(), TranslationCheck.class.getSimpleName());
216                 throw new IllegalArgumentException(exceptionMessage);
217             }
218         }
219     }
220 
221     /**
222      * Checks whether user specified language code is correct (is contained in available locales).
223      * @param userSpecifiedLanguageCode user specified language code.
224      * @return true if user specified language code is correct.
225      */
226     private static boolean isValidLanguageCode(final String userSpecifiedLanguageCode) {
227         boolean valid = false;
228         final Locale[] locales = Locale.getAvailableLocales();
229         for (Locale locale : locales) {
230             if (userSpecifiedLanguageCode.equals(locale.toString())) {
231                 valid = true;
232                 break;
233             }
234         }
235         return valid;
236     }
237 
238     @Override
239     public void beginProcessing(String charset) {
240         filesToProcess.clear();
241     }
242 
243     @Override
244     protected void processFiltered(File file, FileText fileText) {
245         // We just collecting files for processing at finishProcessing()
246         filesToProcess.add(file);
247     }
248 
249     @Override
250     public void finishProcessing() {
251         final Set<ResourceBundle> bundles = groupFilesIntoBundles(filesToProcess, baseName);
252         for (ResourceBundle currentBundle : bundles) {
253             checkExistenceOfDefaultTranslation(currentBundle);
254             checkExistenceOfRequiredTranslations(currentBundle);
255             checkTranslationKeys(currentBundle);
256         }
257     }
258 
259     /**
260      * Checks an existence of default translation file in the resource bundle.
261      * @param bundle resource bundle.
262      */
263     private void checkExistenceOfDefaultTranslation(ResourceBundle bundle) {
264         final Optional<String> fileName = getMissingFileName(bundle, null);
265         if (fileName.isPresent()) {
266             logMissingTranslation(bundle.getPath(), fileName.get());
267         }
268     }
269 
270     /**
271      * Checks an existence of translation files in the resource bundle.
272      * The name of translation file begins with the base name of resource bundle which is followed
273      * by '_' and a language code (country and variant are optional), it ends with the extension
274      * suffix.
275      * @param bundle resource bundle.
276      */
277     private void checkExistenceOfRequiredTranslations(ResourceBundle bundle) {
278         for (String languageCode : requiredTranslations) {
279             final Optional<String> fileName = getMissingFileName(bundle, languageCode);
280             if (fileName.isPresent()) {
281                 logMissingTranslation(bundle.getPath(), fileName.get());
282             }
283         }
284     }
285 
286     /**
287      * Returns the name of translation file which is absent in resource bundle or Guava's Optional,
288      * if there is not missing translation.
289      * @param bundle resource bundle.
290      * @param languageCode language code.
291      * @return the name of translation file which is absent in resource bundle or Guava's Optional,
292      *         if there is not missing translation.
293      */
294     private static Optional<String> getMissingFileName(ResourceBundle bundle, String languageCode) {
295         final String fileNameRegexp;
296         final boolean searchForDefaultTranslation;
297         final String extension = bundle.getExtension();
298         final String baseName = bundle.getBaseName();
299         if (languageCode == null) {
300             searchForDefaultTranslation = true;
301             fileNameRegexp = String.format(Locale.ROOT, REGEXP_FORMAT_TO_CHECK_DEFAULT_TRANSLATIONS,
302                     baseName, extension);
303         }
304         else {
305             searchForDefaultTranslation = false;
306             fileNameRegexp = String.format(Locale.ROOT,
307                 REGEXP_FORMAT_TO_CHECK_REQUIRED_TRANSLATIONS, baseName, languageCode, extension);
308         }
309         Optional<String> missingFileName = Optional.empty();
310         if (!bundle.containsFile(fileNameRegexp)) {
311             if (searchForDefaultTranslation) {
312                 missingFileName = Optional.of(String.format(Locale.ROOT,
313                         DEFAULT_TRANSLATION_FILE_NAME_FORMATTER, baseName, extension));
314             }
315             else {
316                 missingFileName = Optional.of(String.format(Locale.ROOT,
317                         FILE_NAME_WITH_LANGUAGE_CODE_FORMATTER, baseName, languageCode, extension));
318             }
319         }
320         return missingFileName;
321     }
322 
323     /**
324      * Logs that translation file is missing.
325      * @param filePath file path.
326      * @param fileName file name.
327      */
328     private void logMissingTranslation(String filePath, String fileName) {
329         final MessageDispatcher dispatcher = getMessageDispatcher();
330         dispatcher.fireFileStarted(filePath);
331         log(0, MSG_KEY_MISSING_TRANSLATION_FILE, fileName);
332         fireErrors(filePath);
333         dispatcher.fireFileFinished(filePath);
334     }
335 
336     /**
337      * Groups a set of files into bundles.
338      * Only files, which names match base name regexp pattern will be grouped.
339      * @param files set of files.
340      * @param baseNameRegexp base name regexp pattern.
341      * @return set of ResourceBundles.
342      */
343     private static Set<ResourceBundle> groupFilesIntoBundles(Set<File> files,
344                                                              Pattern baseNameRegexp) {
345         final Set<ResourceBundle> resourceBundles = new HashSet<>();
346         for (File currentFile : files) {
347             final String fileName = currentFile.getName();
348             final String baseName = extractBaseName(fileName);
349             final Matcher baseNameMatcher = baseNameRegexp.matcher(baseName);
350             if (baseNameMatcher.matches()) {
351                 final String extension = CommonUtils.getFileExtension(fileName);
352                 final String path = getPath(currentFile.getAbsolutePath());
353                 final ResourceBundle newBundle = new ResourceBundle(baseName, path, extension);
354                 final Optional<ResourceBundle> bundle = findBundle(resourceBundles, newBundle);
355                 if (bundle.isPresent()) {
356                     bundle.get().addFile(currentFile);
357                 }
358                 else {
359                     newBundle.addFile(currentFile);
360                     resourceBundles.add(newBundle);
361                 }
362             }
363         }
364         return resourceBundles;
365     }
366 
367     /**
368      * Searches for specific resource bundle in a set of resource bundles.
369      * @param bundles set of resource bundles.
370      * @param targetBundle target bundle to search for.
371      * @return Guava's Optional of resource bundle (present if target bundle is found).
372      */
373     private static Optional<ResourceBundle> findBundle(Set<ResourceBundle> bundles,
374                                                        ResourceBundle targetBundle) {
375         Optional<ResourceBundle> result = Optional.empty();
376         for (ResourceBundle currentBundle : bundles) {
377             if (targetBundle.getBaseName().equals(currentBundle.getBaseName())
378                     && targetBundle.getExtension().equals(currentBundle.getExtension())
379                     && targetBundle.getPath().equals(currentBundle.getPath())) {
380                 result = Optional.of(currentBundle);
381                 break;
382             }
383         }
384         return result;
385     }
386 
387     /**
388      * Extracts the base name (the unique prefix) of resource bundle from translation file name.
389      * For example "messages" is the base name of "messages.properties",
390      * "messages_de_AT.properties", "messages_en.properties", etc.
391      * @param fileName the fully qualified name of the translation file.
392      * @return the extracted base name.
393      */
394     private static String extractBaseName(String fileName) {
395         final String regexp;
396         final Matcher languageCountryVariantMatcher =
397             LANGUAGE_COUNTRY_VARIANT_PATTERN.matcher(fileName);
398         final Matcher languageCountryMatcher = LANGUAGE_COUNTRY_PATTERN.matcher(fileName);
399         final Matcher languageMatcher = LANGUAGE_PATTERN.matcher(fileName);
400         if (languageCountryVariantMatcher.matches()) {
401             regexp = LANGUAGE_COUNTRY_VARIANT_PATTERN.pattern();
402         }
403         else if (languageCountryMatcher.matches()) {
404             regexp = LANGUAGE_COUNTRY_PATTERN.pattern();
405         }
406         else if (languageMatcher.matches()) {
407             regexp = LANGUAGE_PATTERN.pattern();
408         }
409         else {
410             regexp = DEFAULT_TRANSLATION_REGEXP;
411         }
412         // We use substring(...) instead of replace(...), so that the regular expression does
413         // not have to be compiled each time it is used inside 'replace' method.
414         final String removePattern = regexp.substring("^.+".length(), regexp.length());
415         return fileName.replaceAll(removePattern, "");
416     }
417 
418     /**
419      * Extracts path from a file name which contains the path.
420      * For example, if file nam is /xyz/messages.properties, then the method
421      * will return /xyz/.
422      * @param fileNameWithPath file name which contains the path.
423      * @return file path.
424      */
425     private static String getPath(String fileNameWithPath) {
426         return fileNameWithPath
427             .substring(0, fileNameWithPath.lastIndexOf(File.separator));
428     }
429 
430     /**
431      * Checks resource files in bundle for consistency regarding their keys.
432      * All files in bundle must have the same key set. If this is not the case
433      * an error message is posted giving information which key misses in which file.
434      * @param bundle resource bundle.
435      */
436     private void checkTranslationKeys(ResourceBundle bundle) {
437         final Set<File> filesInBundle = bundle.getFiles();
438         if (filesInBundle.size() >= 2) {
439             // build a map from files to the keys they contain
440             final Set<String> allTranslationKeys = new HashSet<>();
441             final SetMultimap<File, String> filesAssociatedWithKeys = HashMultimap.create();
442             for (File currentFile : filesInBundle) {
443                 final Set<String> keysInCurrentFile = getTranslationKeys(currentFile);
444                 allTranslationKeys.addAll(keysInCurrentFile);
445                 filesAssociatedWithKeys.putAll(currentFile, keysInCurrentFile);
446             }
447             checkFilesForConsistencyRegardingTheirKeys(filesAssociatedWithKeys, allTranslationKeys);
448         }
449     }
450 
451     /**
452      * Compares th the specified key set with the key sets of the given translation files (arranged
453      * in a map). All missing keys are reported.
454      * @param fileKeys a Map from translation files to their key sets.
455      * @param keysThatMustExist the set of keys to compare with.
456      */
457     private void checkFilesForConsistencyRegardingTheirKeys(SetMultimap<File, String> fileKeys,
458                                                             Set<String> keysThatMustExist) {
459         for (File currentFile : fileKeys.keySet()) {
460             final MessageDispatcher dispatcher = getMessageDispatcher();
461             final String path = currentFile.getPath();
462             dispatcher.fireFileStarted(path);
463             final Set<String> currentFileKeys = fileKeys.get(currentFile);
464             final Set<String> missingKeys = keysThatMustExist.stream()
465                 .filter(e -> !currentFileKeys.contains(e)).collect(Collectors.toSet());
466             if (!missingKeys.isEmpty()) {
467                 for (Object key : missingKeys) {
468                     log(0, MSG_KEY, key);
469                 }
470             }
471             fireErrors(path);
472             dispatcher.fireFileFinished(path);
473         }
474     }
475 
476     /**
477      * Loads the keys from the specified translation file into a set.
478      * @param file translation file.
479      * @return a Set object which holds the loaded keys.
480      */
481     private Set<String> getTranslationKeys(File file) {
482         Set<String> keys = new HashSet<>();
483         InputStream inStream = null;
484         try {
485             inStream = new FileInputStream(file);
486             final Properties translations = new Properties();
487             translations.load(inStream);
488             keys = translations.stringPropertyNames();
489         }
490         catch (final IOException ex) {
491             logIoException(ex, file);
492         }
493         finally {
494             Closeables.closeQuietly(inStream);
495         }
496         return keys;
497     }
498 
499     /**
500      * Helper method to log an io exception.
501      * @param exception the exception that occurred
502      * @param file the file that could not be processed
503      */
504     private void logIoException(IOException exception, File file) {
505         String[] args = null;
506         String key = "general.fileNotFound";
507         if (!(exception instanceof FileNotFoundException)) {
508             args = new String[] {exception.getMessage()};
509             key = "general.exception";
510         }
511         final LocalizedMessage message =
512             new LocalizedMessage(
513                 0,
514                 Definitions.CHECKSTYLE_BUNDLE,
515                 key,
516                 args,
517                 getId(),
518                 getClass(), null);
519         final SortedSet<LocalizedMessage> messages = new TreeSet<>();
520         messages.add(message);
521         getMessageDispatcher().fireErrors(file.getPath(), messages);
522         log.debug("IOException occurred.", exception);
523     }
524 
525     /** Class which represents a resource bundle. */
526     private static class ResourceBundle {
527         /** Bundle base name. */
528         private final String baseName;
529         /** Common extension of files which are included in the resource bundle. */
530         private final String extension;
531         /** Common path of files which are included in the resource bundle. */
532         private final String path;
533         /** Set of files which are included in the resource bundle. */
534         private final Set<File> files;
535 
536         /**
537          * Creates a ResourceBundle object with specific base name, common files extension.
538          * @param baseName bundle base name.
539          * @param path common path of files which are included in the resource bundle.
540          * @param extension common extension of files which are included in the resource bundle.
541          */
542         ResourceBundle(String baseName, String path, String extension) {
543             this.baseName = baseName;
544             this.path = path;
545             this.extension = extension;
546             files = new HashSet<>();
547         }
548 
549         public String getBaseName() {
550             return baseName;
551         }
552 
553         public String getPath() {
554             return path;
555         }
556 
557         public String getExtension() {
558             return extension;
559         }
560 
561         public Set<File> getFiles() {
562             return Collections.unmodifiableSet(files);
563         }
564 
565         /**
566          * Adds a file into resource bundle.
567          * @param file file which should be added into resource bundle.
568          */
569         public void addFile(File file) {
570             files.add(file);
571         }
572 
573         /**
574          * Checks whether a resource bundle contains a file which name matches file name regexp.
575          * @param fileNameRegexp file name regexp.
576          * @return true if a resource bundle contains a file which name matches file name regexp.
577          */
578         public boolean containsFile(String fileNameRegexp) {
579             boolean containsFile = false;
580             for (File currentFile : files) {
581                 if (Pattern.matches(fileNameRegexp, currentFile.getName())) {
582                     containsFile = true;
583                     break;
584                 }
585             }
586             return containsFile;
587         }
588     }
589 }