Orcus
parser_token_buffer.hpp
1/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
2/*
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
6 */
7
8#ifndef INCLUDED_ORCUS_DETAIL_THREAD_PARSER_TOKEN_BUFFER_HPP
9#define INCLUDED_ORCUS_DETAIL_THREAD_PARSER_TOKEN_BUFFER_HPP
10
11#include "orcus/exception.hpp"
12
13#include <mutex>
14#include <condition_variable>
15
16namespace orcus { namespace detail { namespace thread {
17
22template<typename _TokensT>
24{
25 typedef _TokensT tokens_type;
26
27 mutable std::mutex m_mtx_tokens;
28 std::condition_variable m_cv_tokens_empty;
29 std::condition_variable m_cv_tokens_ready;
30
31 tokens_type m_tokens; // token buffer used to hand over tokens to the client.
32
33 size_t m_token_size_threshold;
34 const size_t m_max_token_size;
35
36 bool m_parsing_progress;
37
38 bool tokens_empty() const
39 {
40 std::lock_guard<std::mutex> lock(m_mtx_tokens);
41 return m_tokens.empty();
42 }
43
50 void wait_until_tokens_empty()
51 {
52 std::unique_lock<std::mutex> lock(m_mtx_tokens);
53 while (!m_tokens.empty())
54 m_cv_tokens_empty.wait(lock);
55 }
56
57public:
58 parser_token_buffer(size_t min_token_size, size_t max_token_size) :
59 m_token_size_threshold(std::max<size_t>(min_token_size, 1)),
60 m_max_token_size(max_token_size),
61 m_parsing_progress(true)
62 {
63 if (m_token_size_threshold > m_max_token_size)
65 "initial token size threshold is already larger than the max token size.");
66 }
67
76 void check_and_notify(tokens_type& parser_tokens)
77 {
78 if (parser_tokens.size() < m_token_size_threshold)
79 // Still below the threshold.
80 return;
81
82 if (!tokens_empty())
83 {
84 if (m_token_size_threshold < (m_max_token_size/2))
85 {
86 // Double the threshold and continue to parse.
87 m_token_size_threshold *= 2;
88 return;
89 }
90
91 // We cannot increase the threshold any more. Wait for the
92 // client to finish.
93 wait_until_tokens_empty();
94 }
95
96 std::unique_lock<std::mutex> lock(m_mtx_tokens);
97 m_tokens.swap(parser_tokens);
98 lock.unlock();
99 m_cv_tokens_ready.notify_one();
100 }
101
110 void notify_and_finish(tokens_type& parser_tokens)
111 {
112 // Wait until the client tokens get used up.
113 wait_until_tokens_empty();
114
115 {
116 std::lock_guard<std::mutex> lock(m_mtx_tokens);
117 m_tokens.swap(parser_tokens);
118 m_parsing_progress = false;
119 }
120 m_cv_tokens_ready.notify_one();
121 }
122
133 bool next_tokens(tokens_type& tokens)
134 {
135 tokens.clear();
136
137 // Wait until the parser passes a new set of tokens.
138 std::unique_lock<std::mutex> lock(m_mtx_tokens);
139 while (m_tokens.empty() && m_parsing_progress)
140 m_cv_tokens_ready.wait(lock);
141
142 // Get the new tokens and notify the parser.
143 tokens.swap(m_tokens);
144 bool parsing_progress = m_parsing_progress; // Make a copy so that lock can be released safely.
145
146 lock.unlock();
147
148 m_cv_tokens_empty.notify_one();
149
150 return parsing_progress;
151 }
152
159 size_t token_size_threshold() const
160 {
161 if (m_parsing_progress)
162 return 0;
163
164 return m_token_size_threshold;
165 }
166};
167
168}}}
169
170#endif
171
172/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
Definition: parser_token_buffer.hpp:24
void notify_and_finish(tokens_type &parser_tokens)
Definition: parser_token_buffer.hpp:110
void check_and_notify(tokens_type &parser_tokens)
Definition: parser_token_buffer.hpp:76
size_t token_size_threshold() const
Definition: parser_token_buffer.hpp:159
bool next_tokens(tokens_type &tokens)
Definition: parser_token_buffer.hpp:133
Definition: exception.hpp:34
Definition: tokens.hpp:22