@inproceedings{iter2020conpono,
 abstract = {Recent models for unsupervised representation learning of text have employed a number of techniques to improve contextual word representations but have put little focus on discourse-level representations. We propose Conpono, an inter-sentence objective for pretraining language models that models discourse coherence and the distance between sentences. Given an anchor sentence, our model is trained to predict the text k sentences away using a sampled-softmax objective where the candidates consist of neighboring sentences and sentences randomly sampled from the corpus. On the discourse representation benchmark DiscoEval, our model improves over the previous state-of-the-art by up to 13{\%} and on average 4{\%} absolute across 7 tasks. Our model is the same size as BERT-Base, but outperforms the much larger BERT-Large model and other more recent approaches that incorporate discourse. We also show that Conpono yields gains of 2{\%}-6{\%} absolute even for tasks that do not explicitly evaluate discourse: textual entailment (RTE), common sense reasoning (COPA) and reading comprehension (ReCoRD).},
 address = {Online},
 author = {Iter, Dan  and
Guu, Kelvin  and
Lansing, Larry  and
Jurafsky, Dan},
 booktitle = {Association for Computational Linguistics (ACL)},
 link = {https://www.aclweb.org/anthology/2020.acl-main.439},
 month = {jul},
 pages = {4859--4870},
 publisher = {Association for Computational Linguistics},
 title = {Pretraining with Contrastive Sentence Objectives Improves Discourse Performance of Language Models},
 url = {https://nlp.stanford.edu/pubs/iter2020conpono.pdf},
 year = {2020}
}