@article {pub3983,
	title = {Beyond Cross Validation - Classifier{\textquoteright}s Accuracy Prediction for Benchmarking Incremental and Active Learning Models},
	author = {Christian  Limberg AND Heiko Wersing AND Helge Ritter},
	year = {2020},
	abstract = { For incremental machine-learning applications it is often important to robustly estimate
the system accuracy during training, especially if humans perform the supervised teaching.
Cross-validation and interleaved test/train error are here the standard supervised approaches.
We propose a novel semi-supervised accuracy estimation approach that clearly outperforms these
two methods. We introduce the Configram Estimation (CGEM) approach to predict the accuracy of
any classifier that delivers confidences. By calculating classification confidences for unseen samples,
it is possible to train an offline regression model, capable of predicting the classifier{\textquoteright}s accuracy on
novel data in a semi-supervised fashion. We evaluate our method with several diverse classifiers
and on analytical and real-world benchmark data sets for both incremental and active learning.
The results show that our novel method improves accuracy estimation over standard methods and
requires less supervised training data after deployment of the model. We demonstrate the application
of our approach to a challenging robot object recognition task, where the human teacher can use our
method to judge sufficient training.},
	publisher = {MDPI},
	journal = {Machine Learning and Knowledge Extraction},
	volume = {2},
	pages = {327-346}
}