path: root/core/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/pdom/indexer/PDOMIndexerTask.java

/*******************************************************************************
 * Copyright (c) 2006, 2020 Wind River Systems, Inc. and others.
 *
 * This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License 2.0
 * which accompanies this distribution, and is available at
 * https://www.eclipse.org/legal/epl-2.0/
 *
 * SPDX-License-Identifier: EPL-2.0
 *
 * Contributors:
 *     Markus Schorn - initial API and implementation
 *     Sergey Prigogin (Google)
 *     Marc-Andre Laperle (Ericsson)
 *     Alexander Fedorov (ArSysOp) - Bug 561992
 *******************************************************************************/
package org.eclipse.cdt.internal.core.pdom.indexer;

import java.text.NumberFormat;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

import org.eclipse.cdt.core.CCorePlugin;
import org.eclipse.cdt.core.CCorePreferenceConstants;
import org.eclipse.cdt.core.dom.IPDOMIndexer;
import org.eclipse.cdt.core.dom.IPDOMIndexerTask;
import org.eclipse.cdt.core.index.IIndexManager;
import org.eclipse.cdt.core.model.ICProject;
import org.eclipse.cdt.core.model.ITranslationUnit;
import org.eclipse.cdt.core.parser.IncludeExportPatterns;
import org.eclipse.cdt.internal.core.index.IWritableIndex;
import org.eclipse.cdt.internal.core.index.IWritableIndexManager;
import org.eclipse.cdt.internal.core.model.CProject;
import org.eclipse.cdt.internal.core.pdom.AbstractIndexerTask;
import org.eclipse.cdt.internal.core.pdom.ITodoTaskUpdater;
import org.eclipse.cdt.internal.core.pdom.IndexerProgress;
import org.eclipse.cdt.internal.core.pdom.db.ChunkCache;
import org.eclipse.core.resources.IResource;
import org.eclipse.core.runtime.CoreException;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.IStatus;
import org.eclipse.core.runtime.Platform;
import org.eclipse.core.runtime.Status;
import org.eclipse.osgi.util.NLS;

/**
 * Configures the abstract indexer task suitable for indexing projects.
 */
public abstract class PDOMIndexerTask extends AbstractIndexerTask implements IPDOMIndexerTask {
	private static final String TRUE = Boolean.TRUE.toString();

	private AbstractPDOMIndexer fIndexer;
	private boolean fWriteInfoToLog;

	protected PDOMIndexerTask(ITranslationUnit[] forceFiles, ITranslationUnit[] updateFiles,
			ITranslationUnit[] removeFiles, AbstractPDOMIndexer indexer, boolean isFastIndexer) {
		super(concat(forceFiles, updateFiles), removeFiles, new ProjectIndexerInputAdapter(indexer.getProject()),
				isFastIndexer);
		fIndexer = indexer;
		setShowActivity(checkDebugOption(TRACE_ACTIVITY, TRUE));
		setShowInclusionProblems(checkDebugOption(TRACE_INCLUSION_PROBLEMS, TRUE));
		setShowScannerProblems(checkDebugOption(TRACE_SCANNER_PROBLEMS, TRUE));
		setShowSyntaxProblems(checkDebugOption(TRACE_SYNTAX_PROBLEMS, TRUE));
		setShowProblems(checkDebugOption(TRACE_PROBLEMS, TRUE));
		final long fileLimit = getIntProperty(IndexerPreferences.KEY_SKIP_FILES_LARGER_THAN_MB, 0);
		final long includedFileLimit = getIntProperty(IndexerPreferences.KEY_SKIP_INCLUDED_FILES_LARGER_THAN_MB, 0);
		setFileSizeLimits(fileLimit * 1024 * 1024, includedFileLimit * 1024 * 1024);
		setIndexAllHeaderVersions(checkProperty(IndexerPreferences.KEY_INDEX_ALL_HEADER_VERSIONS));
		setHeadersToIndexAllVersions(getStringSet(IndexerPreferences.KEY_INDEX_ALL_VERSIONS_SPECIFIC_HEADERS));
		if (checkProperty(IndexerPreferences.KEY_SKIP_ALL_REFERENCES)) {
			setSkipReferences(SKIP_ALL_REFERENCES);
		} else {
			int skipRefs = 0;
			if (checkProperty(IndexerPreferences.KEY_SKIP_IMPLICIT_REFERENCES)) {
				skipRefs |= SKIP_IMPLICIT_REFERENCES;
			}
			if (checkProperty(IndexerPreferences.KEY_SKIP_TYPE_REFERENCES)) {
				skipRefs |= SKIP_TYPE_REFERENCES;
			}
			if (checkProperty(IndexerPreferences.KEY_SKIP_MACRO_REFERENCES)) {
				skipRefs |= SKIP_MACRO_REFERENCES;
			}
			if (skipRefs != 0) {
				setSkipReferences(skipRefs);
			}
		}
		if (checkProperty(IndexerPreferences.KEY_INDEX_ALL_FILES)) {
			setIndexFilesWithoutBuildConfiguration(true);
			boolean i1 = checkProperty(IndexerPreferences.KEY_INDEX_UNUSED_HEADERS_WITH_DEFAULT_LANG);
			boolean i2 = checkProperty(IndexerPreferences.KEY_INDEX_UNUSED_HEADERS_WITH_ALTERNATE_LANG);
			UnusedHeaderStrategy strategy;
			if (i1 == i2) {
				strategy = i1 ? UnusedHeaderStrategy.useBoth : UnusedHeaderStrategy.skip;
			} else {
				strategy = i1 == CProject.hasCCNature(getCProject().getProject()) ? UnusedHeaderStrategy.useCPP
						: UnusedHeaderStrategy.useC;
			}
			setIndexHeadersWithoutContext(strategy);
		} else {
			setIndexFilesWithoutBuildConfiguration(false);
			setIndexHeadersWithoutContext(UnusedHeaderStrategy.skip);
		}
		setUpdateFlags(IIndexManager.UPDATE_CHECK_TIMESTAMPS | IIndexManager.UPDATE_CHECK_CONTENTS_HASH);
		setForceFirstFiles(forceFiles.length);

		ICProject project = getCProject();
		String privatePattern = CCorePreferenceConstants.getPreference(CCorePreferenceConstants.INCLUDE_PRIVATE_PATTERN,
				project, null);
		if (privatePattern != null) {
			try {
				setPragmaPrivatePattern(Pattern.compile(privatePattern));
			} catch (PatternSyntaxException e) {
				CCorePlugin.log(e);
			}
		}
	}

	private static ITranslationUnit[] concat(ITranslationUnit[] added, ITranslationUnit[] changed) {
		HashSet<ITranslationUnit> union = new HashSet<>(added.length + changed.length);
		union.addAll(Arrays.asList(added));
		union.addAll(Arrays.asList(changed));
		final ITranslationUnit[] result = union.toArray(new ITranslationUnit[union.size()]);
		Arrays.sort(result, new Comparator<ITranslationUnit>() {
			@Override
			public int compare(ITranslationUnit o1, ITranslationUnit o2) {
				IResource res1 = o1.getResource();
				IResource res2 = o2.getResource();
				if (res1 != null && res2 != null) {
					return compare(res1.getFullPath().segments(), res2.getFullPath().segments());
				}
				return res1 != null ? -1 : res2 != null ? 1 : 0;
			}

			private int compare(String[] s1, String[] s2) {
				int max = Math.min(s1.length, s2.length) - 1;
				for (int i = 0; i < max; i++) {
					int cmp = s1[i].compareTo(s2[i]);
					if (cmp != 0)
						return cmp;
				}
				int cmp = s1.length - s2.length;
				if (cmp != 0)
					return cmp;
				return s1[max].compareTo(s2[max]);
			}
		});
		return result;
	}

	@Override
	public final IPDOMIndexer getIndexer() {
		return fIndexer;
	}

	@Override
	public final void run(IProgressMonitor monitor) throws InterruptedException {
		long start = System.currentTimeMillis();
		runTask(monitor);
		traceEnd(start, fIndex, monitor.isCanceled());
	}

	/**
	 * Checks whether a given debug option is enabled. See {@link IPDOMIndexerTask}
	 * for valid values.
	 * @since 4.0
	 */
	public static boolean checkDebugOption(String option, String value) {
		String trace = Platform.getDebugOption(option);
		boolean internallyActivated = Boolean.getBoolean(option);
		return internallyActivated || (trace != null && trace.equalsIgnoreCase(value));
	}

	private boolean checkProperty(String key) {
		return TRUE.equals(getIndexer().getProperty(key));
	}

	private Set<String> getStringSet(String key) {
		String prefSetting = getIndexer().getProperty(key);
		if (prefSetting != null && !prefSetting.isEmpty()) {
			return new HashSet<>(Arrays.asList(prefSetting.split(","))); //$NON-NLS-1$
		}

		return Collections.emptySet();
	}

	private int getIntProperty(String key, int defaultValue) {
		final String value = getIndexer().getProperty(key);
		if (value != null) {
			try {
				return Integer.parseInt(value);
			} catch (NumberFormatException e) {
			}
		}
		return defaultValue;
	}

	@Override
	protected final IWritableIndex createIndex() {
		try {
			return ((IWritableIndexManager) CCorePlugin.getIndexManager()).getWritableIndex(getCProject());
		} catch (CoreException e) {
			CCorePlugin.log(e);
		}
		return null;
	}

	@Override
	protected final ITodoTaskUpdater createTodoTaskUpdater() {
		return new TodoTaskUpdater();
	}

	@Override
	protected final IncludeExportPatterns getIncludeExportPatterns() {
		ICProject project = getCProject();
		String exportPattern = CCorePreferenceConstants.getPreference(CCorePreferenceConstants.INCLUDE_EXPORT_PATTERN,
				project, null);
		String beginExportsPattern = CCorePreferenceConstants
				.getPreference(CCorePreferenceConstants.INCLUDE_BEGIN_EXPORTS_PATTERN, project, null);
		String endExportsPattern = CCorePreferenceConstants
				.getPreference(CCorePreferenceConstants.INCLUDE_END_EXPORTS_PATTERN, project, null);
		if (exportPattern == null && beginExportsPattern == null && endExportsPattern == null)
			return null;
		return new IncludeExportPatterns(exportPattern, beginExportsPattern, endExportsPattern);
	}

	protected void traceEnd(long start, IWritableIndex index, boolean wasCancelled) {
		// log entry
		if (fWriteInfoToLog && !wasCancelled && index != null) {
			final long totalTime = System.currentTimeMillis() - start;
			final IndexerProgress info = getProgressInformation();
			final int sum = fStatistics.fDeclarationCount + fStatistics.fReferenceCount
					+ fStatistics.fProblemBindingCount;
			final double problemPct = sum == 0 ? 0.0 : (double) fStatistics.fProblemBindingCount / (double) sum;
			NumberFormat nfGroup = NumberFormat.getNumberInstance();
			nfGroup.setGroupingUsed(true);
			NumberFormat nfPercent = NumberFormat.getPercentInstance();
			int fractionalDigits = Math.max(1 - (int) Math.floor(Math.log10(problemPct * 100.)), 0);
			nfPercent.setMaximumFractionDigits(fractionalDigits);
			nfPercent.setMinimumFractionDigits(0);
			NumberFormat nfTime = NumberFormat.getNumberInstance();
			fractionalDigits = Math.max(2 - (int) Math.floor(Math.log10(totalTime / 1000.)), 0);
			nfTime.setMaximumFractionDigits(fractionalDigits);
			nfTime.setMinimumFractionDigits(0);
			nfTime.setGroupingUsed(true);
			final String msg = NLS.bind(Messages.PDOMIndexerTask_indexerInfo,
					new Object[] { getCProject().getElementName(), nfGroup.format(info.fCompletedSources),
							nfGroup.format(info.fCompletedHeaders), nfTime.format(totalTime / 1000.),
							nfGroup.format(fStatistics.fDeclarationCount), nfGroup.format(fStatistics.fReferenceCount),
							nfGroup.format(fStatistics.fUnresolvedIncludesCount),
							nfGroup.format(fStatistics.fPreprocessorProblemCount + fStatistics.fSyntaxProblemsCount),
							nfGroup.format(fStatistics.fProblemBindingCount), nfPercent.format(problemPct) });
			CCorePlugin.getDefault().getLog().log(new Status(IStatus.INFO, CCorePlugin.PLUGIN_ID, msg));
		}

		// tracing
		if (checkDebugOption(IPDOMIndexerTask.TRACE_STATISTICS, TRUE)) {
			String indent = "   "; //$NON-NLS-1$
			final long totalTime = System.currentTimeMillis() - start;
			final IndexerProgress info = getProgressInformation();
			final int sum = fStatistics.fDeclarationCount + fStatistics.fReferenceCount
					+ fStatistics.fProblemBindingCount;
			final double problemPct = sum == 0 ? 0.0 : (double) fStatistics.fProblemBindingCount / (double) sum;
			String kind = getIndexer().getClass().getName();
			kind = kind.substring(kind.lastIndexOf('.') + 1);
			final long dbSize = index.getDatabaseSizeBytes();

			System.out.println("C/C++ Indexer: Project '" + getCProject().getElementName() //$NON-NLS-1$
					+ "' (" + info.fCompletedSources + " sources, " //$NON-NLS-1$//$NON-NLS-2$
					+ info.fCompletedHeaders + " headers)"); //$NON-NLS-1$
			boolean skipRefs = checkProperty(IndexerPreferences.KEY_SKIP_ALL_REFERENCES);
			boolean skipImplRefs = skipRefs || checkProperty(IndexerPreferences.KEY_SKIP_IMPLICIT_REFERENCES);
			boolean skipTypeRefs = skipRefs || checkProperty(IndexerPreferences.KEY_SKIP_TYPE_REFERENCES);
			boolean skipMacroRefs = skipRefs || checkProperty(IndexerPreferences.KEY_SKIP_MACRO_REFERENCES);
			System.out.println(indent + " Options: " //$NON-NLS-1$
					+ "indexer='" + kind //$NON-NLS-1$
					+ "', parseAllFiles=" + indexFilesWithoutConfiguration() //$NON-NLS-1$
					+ ", unusedHeaders=" + getIndexHeadersWithoutContext() //$NON-NLS-1$
					+ ", skipReferences=" + skipRefs //$NON-NLS-1$
					+ ", skipImplicitReferences=" + skipImplRefs //$NON-NLS-1$
					+ ", skipTypeReferences=" + skipTypeRefs //$NON-NLS-1$
					+ ", skipMacroReferences=" + skipMacroRefs //$NON-NLS-1$
					+ "."); //$NON-NLS-1$
			System.out.println(indent + " Database: " + dbSize + " bytes"); //$NON-NLS-1$ //$NON-NLS-2$
			System.out.println(indent + " Timings: " //$NON-NLS-1$
					+ totalTime + " total, " //$NON-NLS-1$
					+ fStatistics.fParsingTime + " parser, " //$NON-NLS-1$
					+ fStatistics.fResolutionTime + " resolution, " //$NON-NLS-1$
					+ fStatistics.fAddToIndexTime + " index update."); //$NON-NLS-1$
			System.out.println(indent + " Errors: " //$NON-NLS-1$
					+ fStatistics.fErrorCount + " internal, " //$NON-NLS-1$
					+ fStatistics.fUnresolvedIncludesCount + " include, " //$NON-NLS-1$
					+ fStatistics.fPreprocessorProblemCount + " scanner, " //$NON-NLS-1$
					+ fStatistics.fSyntaxProblemsCount + " syntax errors."); //$NON-NLS-1$
			if (fStatistics.fTooManyTokensCount > 0)
				System.out.println(indent + " Tokens: " //$NON-NLS-1$
						+ fStatistics.fTooManyTokensCount + " TUs with too many tokens."); //$NON-NLS-1$

			NumberFormat nfPercent = NumberFormat.getPercentInstance();
			nfPercent.setMaximumFractionDigits(2);
			nfPercent.setMinimumFractionDigits(2);
			System.out.println(indent + " Names: " //$NON-NLS-1$
					+ fStatistics.fDeclarationCount + " declarations, " //$NON-NLS-1$
					+ fStatistics.fReferenceCount + " references, " //$NON-NLS-1$
					+ fStatistics.fProblemBindingCount + "(" + nfPercent.format(problemPct) + ") unresolved."); //$NON-NLS-1$ //$NON-NLS-2$

			long misses = index.getCacheMisses();
			long hits = index.getCacheHits();
			long tries = misses + hits;
			double missPct = tries == 0 ? 0.0 : (double) misses / (double) tries;
			System.out.println(indent + " Cache[" //$NON-NLS-1$
					+ ChunkCache.getSharedInstance().getMaxSize() / 1024 / 1024 + "MB]: " + //$NON-NLS-1$
					+hits + " hits, " //$NON-NLS-1$
					+ misses + "(" + nfPercent.format(missPct) + ") misses."); //$NON-NLS-1$ //$NON-NLS-2$

			if (Boolean.parseBoolean(System.getProperty("SHOW_COMPRESSED_INDEXER_INFO"))) { //$NON-NLS-1$
				Calendar cal = Calendar.getInstance();
				NumberFormat twoDigits = NumberFormat.getNumberInstance();
				twoDigits.setMinimumIntegerDigits(2);
				NumberFormat nfGroup = NumberFormat.getNumberInstance();
				nfGroup.setGroupingUsed(true);

				final String sep0 = "|"; //$NON-NLS-1$
				final String sep = "|  "; //$NON-NLS-1$
				final String sec = "s"; //$NON-NLS-1$
				final String mb = "MB"; //$NON-NLS-1$
				final String million = "M"; //$NON-NLS-1$
				System.out.print(sep0);
				System.out.print(cal.get(Calendar.YEAR) + twoDigits.format(cal.get(Calendar.MONTH) + 1)
						+ twoDigits.format(cal.get(Calendar.DAY_OF_MONTH)));
				System.out.print(sep);
				System.out.print(nfGroup.format(info.fCompletedSources));
				System.out.print(sep);
				System.out.print(nfGroup.format(info.fCompletedHeaders));
				System.out.print(sep);
				System.out.print(nfGroup.format((totalTime + 500) / 1000) + sec);
				System.out.print(sep);
				System.out.print(nfGroup.format((fStatistics.fParsingTime + 500) / 1000) + sec);
				System.out.print(sep);
				System.out.print(nfGroup.format((fStatistics.fResolutionTime + 500) / 1000) + sec);
				System.out.print(sep);
				System.out.print(nfGroup.format((fStatistics.fAddToIndexTime + 500) / 1000) + sec);
				System.out.print(sep);
				System.out.print(nfGroup.format((dbSize + 1024 * 512) / 1024 / 1024) + mb);
				System.out.print(sep);
				System.out.print(nfGroup.format((tries + 1000 * 500) / 1000000) + million);
				System.out.print(sep);
				System.out.print(nfGroup.format(fStatistics.fDeclarationCount));
				System.out.print(sep);
				System.out.print(nfGroup.format(fStatistics.fReferenceCount));
				System.out.print(sep);
				System.out.print(nfGroup.format(fStatistics.fProblemBindingCount));
				System.out.print(sep);
				System.out.print(nfPercent.format(problemPct));
				System.out.print(sep);
				System.out.print(nfGroup.format(fStatistics.fErrorCount));
				System.out.print(sep);
				System.out.print(nfGroup.format(fStatistics.fUnresolvedIncludesCount));
				System.out.print(sep);
				System.out.print(nfGroup.format(fStatistics.fPreprocessorProblemCount));
				System.out.print(sep);
				System.out.print(nfGroup.format(fStatistics.fSyntaxProblemsCount));
				System.out.println(sep0);
			}
		}
	}

	protected ICProject getCProject() {
		return fIndexer.project;
	}

	public void setWriteInfoToLog() {
		fWriteInfoToLog = true;
	}

	@Override
	public synchronized boolean acceptUrgentTask(IPDOMIndexerTask urgentTask) {
		final IPDOMIndexer ti = urgentTask.getIndexer();
		if (fIndexer == null || ti == null || fIndexer.getProject().getProject() != ti.getProject().getProject()) {
			return false;
		}
		return super.acceptUrgentTask(urgentTask);
	}
}