1 files changed, 55 insertions, 0 deletions
diff --git a/debian/uncrustify-trinity/uncrustify-trinity-0.74.0/src/tokenize.h b/debian/uncrustify-trinity/uncrustify-trinity-0.74.0/src/tokenize.h
new file mode 100644
index 00000000..896dd511
--- /dev/null
+++ b/debian/uncrustify-trinity/uncrustify-trinity-0.74.0/src/tokenize.h
@@ -0,0 +1,55 @@
+/**
+ * @file tokenize.h
+ * prototypes for tokenize.c
+ *
+ * @author  Ben Gardner
+ * @license GPL v2+
+ */
+
+#ifndef TOKENIZE_H_INCLUDED
+#define TOKENIZE_H_INCLUDED
+
+#include "uncrustify_types.h"
+
+
+/**
+ * Test the input string to see if it satisfies the criteria
+ * specified by the disable_processing_cmt option
+ * @param  text      the string to which a match will be attempted
+ * @param  start_idx the starting index within the string from which the
+ *                   search will be performed
+ * @return           returns a non-negative position index that points to the beginning
+ *                   of the line containing the marker, if found
+ */
+int find_disable_processing_comment_marker(const unc_text &text, std::size_t start_idx = 0);
+
+
+/**
+ * Test the input string to see if it satisfies the criteria
+ * specified by the enable_processing_cmt option
+ * @param  text      the string to which a match will be attempted
+ * @param  start_idx the starting index within the string from which the
+ *                   search will be performed
+ * @return           returns a non-negative position index that points to the end
+ *                   of the line containing the marker, if found
+ */
+int find_enable_processing_comment_marker(const unc_text &text, std::size_t start_idx = 0);
+
+
+/**
+ * @brief Parse the text into chunks
+ *
+ * This function parses or tokenizes the whole buffer into a list.
+ * It has to do some tricks to parse preprocessors.
+ *
+ * If output_text() were called immediately after, two things would happen:
+ *  - trailing whitespace are removed.
+ *  - leading space & tabs are converted to the appropriate format.
+ *
+ * All the tokens are inserted before ref. If ref is NULL, they are inserted
+ * at the end of the list.  Line numbers are relative to the start of the data.
+ */
+void tokenize(const std::deque<int> &data, chunk_t *ref);
+
+
+#endif /* TOKENIZE_H_INCLUDED */