1berry/qtbase
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Example: update wordcount example

Browse Source 
Added a QFileDialog to let the user select a path. Before, the path
was statically assigned with "../../" , which is not optimal.
I also modified the findFiles function to check for text files in
general and not only *.cpp and *.h files. Lastly the result of the
word counting is now displayed on the console, as I think this is an
informative output from this example.

Task-number: QTBUG-111165
Change-Id: Ie27c6acb4f79a78e3bef141edb92de08901fde71
Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>
(cherry picked from commit d795dfaee7093a30b278e6e8b9dea30c2e7a2106)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>
This commit is contained in:
Dennis Oberst 2023-02-28 18:47:05 +01:00 committed by Qt Cherry-pick Bot
parent 76c44cd6e1
commit 9cab7e55d8
3 changed files with 124 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
doc/global/manifest-meta.qdocconf
View File

@ -253,7 +253,7 @@ manifestmeta.thumbnail.names = "QtCore/Contiguous Cache Example" \
"QtCore/Semaphores Example" \
"QtCore/Wait Conditions Example" \
"QtConcurrent/Map Example" \
"QtConcurrent/QtConcurrent Word Count Example" \
"QtConcurrent/Word Count" \
"QtConcurrent/Run Function Example" \
"QtGui/Raster Window Example" \
"QtNetwork/Network Download*" \

43
examples/qtconcurrent/wordcount/doc/src/qtconcurrent-wordcount.qdoc
View File

@ -1,15 +1,44 @@
// Copyright (C) 2016 The Qt Company Ltd.
// Copyright (C) 2023 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GFDL-1.3-no-invariants-only
/*!
\example wordcount
\title QtConcurrent Word Count Example
\brief Demonstrates how to use the map-reduce algorithm.
\meta tags {threads, console}
\title Word Count
\ingroup qtconcurrentexamples
\brief Demonstrates how to use the map-reduce algorithm.
The QtConcurrent Word Count example demonstrates the use of the map-reduce
algorithm when applied to the problem of counting words in a collection
of files.
The Qt Concurrent \e {Word Count} example demonstrates the use of the
map-reduce algorithm when applied to the problem of counting words in a
collection of files.
This is a command-line application.
First, the Application starts a QFileDialog to select a starting
path, and then prints the output to the console.
\include examples-run.qdocinc
\section1 Comparing the operations
Compare a single-threaded, sequential approach to counting the words in
the text files to a multithreaded approach with mappedReduced():
\dots
\snippet wordcount/main.cpp 1
\dots
\snippet wordcount/main.cpp 2
\dots
The first argument to the \l {QtConcurrent::}{mappedReduced} function is the
container to operate on. The second argument is the mapping function
\c {countWords()}. It is called in parallel by multiple threads. The
third argument is the reducing function \c {reduce()}. It is called
once for each result returned by the mapping function, and generates the
final computation result.
The function returns a QFuture object of type \c WordCount. Call the
\l {QFuture::}{result} function immediately on this QFuture to block further
execution until the result becomes available.
\note The mapping function must be thread-safe since it is called from
multiple threads.
*/

140
examples/qtconcurrent/wordcount/main.cpp
View File

@ -1,43 +1,18 @@
// Copyright (C) 2016 The Qt Company Ltd.
// Copyright (C) 2023 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR BSD-3-Clause
#include <QList>
#include <QMap>
#include <QTextStream>
#include <QString>
#include <QStringList>
#include <QDir>
#include <QElapsedTimer>
#include <QApplication>
#include <QDebug>
#include <qtconcurrentmap.h>
using namespace QtConcurrent;
/*
Utility function that recursivily searches for files.
*/
QStringList findFiles(const QString &startDir, const QStringList &filters)
{
QStringList names;
QDir dir(startDir);
const auto files = dir.entryList(filters, QDir::Files);
for (const QString &file : files)
names += startDir + '/' + file;
const auto subdirs = dir.entryList(QDir::AllDirs | QDir::NoDotAndDotDot);
for (const QString &subdir : subdirs)
names += findFiles(startDir + '/' + subdir, filters);
return names;
}
#include <QtWidgets/qfiledialog.h>
#include <QtWidgets/qapplication.h>
#include <QtCore/qmimedatabase.h>
#include <QtCore/qelapsedtimer.h>
#include <QtConcurrent/qtconcurrentmap.h>
typedef QMap<QString, int> WordCount;
/*
Single threaded word counter function.
*/
void printHighestResult(const WordCount &, qsizetype);
QStringList findFiles(const QString &);
// Single threaded word counter function.
WordCount singleThreadedWordCount(const QStringList &files)
{
WordCount wordCount;
@ -46,7 +21,7 @@ WordCount singleThreadedWordCount(const QStringList &files)
f.open(QIODevice::ReadOnly);
QTextStream textStream(&f);
while (!textStream.atEnd()) {
const auto words = textStream.readLine().split(' ');
const auto words = textStream.readLine().split(' ', Qt::SkipEmptyParts);
for (const QString &word : words)
wordCount[word] += 1;
}
@ -54,7 +29,6 @@ WordCount singleThreadedWordCount(const QStringList &files)
return wordCount;
}
// countWords counts the words in a single file. This function is
// called in parallel by several threads and must be thread
// safe.
@ -66,7 +40,7 @@ WordCount countWords(const QString &file)
WordCount wordCount;
while (!textStream.atEnd()) {
const auto words = textStream.readLine().split(' ');
const auto words = textStream.readLine().split(' ', Qt::SkipEmptyParts);
for (const QString &word : words)
wordCount[word] += 1;
}
@ -86,33 +60,93 @@ void reduce(WordCount &result, const WordCount &w)
int main(int argc, char** argv)
{
QApplication app(argc, argv);
qDebug() << "finding files...";
QStringList files = findFiles("../../", QStringList() << "*.cpp" << "*.h");
qDebug() << files.count() << "files";
app.setOrganizationName("QtProject");
app.setApplicationName(QCoreApplication::translate("main", "Word Count"));
qDebug() << "warmup";
{
WordCount total = singleThreadedWordCount(files);
}
QFileDialog fileDialog;
fileDialog.setOption(QFileDialog::ReadOnly);
// Grab the directory path from the dialog
auto dirPath = QFileDialog::getExistingDirectory(nullptr,
QCoreApplication::translate("main","Select a Folder"),
QDir::currentPath());
qDebug() << "warmup done";
QStringList files = findFiles(dirPath);
qDebug() << QCoreApplication::translate("main", "Indexing %1 files in %2")
.arg(files.size()).arg(dirPath);
int singleThreadTime = 0;
// Start the single threaded operation
qint64 singleThreadTime;
{
QElapsedTimer timer;
timer.start();
//! [1]
WordCount total = singleThreadedWordCount(files);
//! [1]
singleThreadTime = timer.elapsed();
qDebug() << "single thread" << singleThreadTime;
qDebug() << QCoreApplication::translate("main", "Single threaded scanning took %1 ms")
.arg(singleThreadTime);
}
int mapReduceTime = 0;
// Start the multithreaded mappedReduced operation.
qint64 mapReduceTime;
{
QElapsedTimer timer;
timer.start();
WordCount total = mappedReduced(files, countWords, reduce).result();
//! [2]
WordCount total = QtConcurrent::mappedReduced(files, countWords, reduce).result();
//! [2]
mapReduceTime = timer.elapsed();
qDebug() << "MapReduce" << mapReduceTime;
qDebug() << QCoreApplication::translate("main", "MapReduce scanning took %1 ms")
.arg(mapReduceTime);
qDebug() << QCoreApplication::translate("main", "MapReduce speedup: %1")
.arg(((double)singleThreadTime - (double)mapReduceTime) / (double)mapReduceTime + 1);
printHighestResult(total, 10);
}
qDebug() << "MapReduce speedup x" << ((double)singleThreadTime - (double)mapReduceTime) / (double)mapReduceTime + 1;
}
// Utility function that recursively searches for text files.
QStringList findFiles(const QString &startDir)
{
QStringList names;
QDir dir(startDir);
static const QMimeDatabase db;
const auto files = dir.entryList(QDir::Files);
for (const QString &file : files) {
const auto path = startDir + QDir::separator() + file;
const QMimeType mime = db.mimeTypeForFile(QFileInfo(path));
const auto mimeTypesForFile = mime.parentMimeTypes();
for (const auto &i : mimeTypesForFile) {
if (i.contains("text", Qt::CaseInsensitive)
|| mime.comment().contains("text", Qt::CaseInsensitive)) {
names += startDir + QDir::separator() + file;
}
}
}
const auto subdirs = dir.entryList(QDir::AllDirs | QDir::NoDotAndDotDot);
for (const QString &subdir : subdirs) {
if (names.length() >= 20000) {
qDebug() << QCoreApplication::translate("main", "Too many files! Aborting ...");
exit(-1);
}
names += findFiles(startDir + QDir::separator() + subdir);
}
return names;
}
// Utility function that prints the results of the map in decreasing order based on the value.
void printHighestResult(const WordCount &countedWords, qsizetype nResults)
{
using pair = QPair<QString, int>;
QList<pair> vec;
std::copy(countedWords.keyValueBegin(), countedWords.keyValueEnd(),
std::back_inserter<QList<pair>>(vec));
std::sort(vec.begin(), vec.end(),
[](const pair &l, const pair &r) { return l.second > r.second; });
qDebug() << QCoreApplication::translate("main", "Most occurring words are:");
for (qsizetype i = 0; i < qMin(vec.size(), nResults); ++i)
qDebug() << vec[i].first << " : " << vec[i].second;
}