Identifying record replicas in digital libraries and
other types of digital repositories is fundamental to
improve the quality of their content and services as
well as to yield eventual sharing efforts. Several
deduplication strategies are available, but most of
them rely on manually chosen settings to combine
evidence used to identify records as being replicas. In
this paper, we present the results of experiments we
have carried out with a novel machine learning approach
we have proposed for the de duplication problem. This
approach, based on genetic programming (GP), is able to
automatically generate similarity functions to identify
record replicas in a given repository. The generated
similarity functions properly combine and weight the
best evidence available among the record fields in
order to tell when two distinct records represent the
same real-world entity. The results of the experiments
show that our approach outperforms the baseline method
by Fellegi and Sunter by more than 12percent when
identifying replicas in a data set containing
researcher's personal data, and by more than 7percent,
in a data set with article citation data
%0 Conference Paper
%1 deCarvalho:2006:JCDL
%A de Carvalho, Moises G.
%A Goncalves, Marcos Andre
%A Laender, Alberto H. F.
%A da Silva, Altigran S.
%B Proceedings of the 6th ACM/IEEE-CS Joint Conference on
Digital Libraries, JCDL '06
%C Chapel Hill, NC, USA
%D 2006
%I IEEE
%K Deduplication, Digital Libraries algorithms, genetic programming,
%P 41--50
%R doi:10.1145/1141753.1141760
%T Learning to deduplicate
%U http://delivery.acm.org/10.1145/1150000/1141760/p41-decarvalho.pdf?key1=1141760&key2=6906456911&coll=GUIDE&dl=GUIDE&CFID=45325455&CFTOKEN=75817203
%X Identifying record replicas in digital libraries and
other types of digital repositories is fundamental to
improve the quality of their content and services as
well as to yield eventual sharing efforts. Several
deduplication strategies are available, but most of
them rely on manually chosen settings to combine
evidence used to identify records as being replicas. In
this paper, we present the results of experiments we
have carried out with a novel machine learning approach
we have proposed for the de duplication problem. This
approach, based on genetic programming (GP), is able to
automatically generate similarity functions to identify
record replicas in a given repository. The generated
similarity functions properly combine and weight the
best evidence available among the record fields in
order to tell when two distinct records represent the
same real-world entity. The results of the experiments
show that our approach outperforms the baseline method
by Fellegi and Sunter by more than 12percent when
identifying replicas in a data set containing
researcher's personal data, and by more than 7percent,
in a data set with article citation data
%@ 1-59593-354-9
@inproceedings{deCarvalho:2006:JCDL,
abstract = {Identifying record replicas in digital libraries and
other types of digital repositories is fundamental to
improve the quality of their content and services as
well as to yield eventual sharing efforts. Several
deduplication strategies are available, but most of
them rely on manually chosen settings to combine
evidence used to identify records as being replicas. In
this paper, we present the results of experiments we
have carried out with a novel machine learning approach
we have proposed for the de duplication problem. This
approach, based on genetic programming (GP), is able to
automatically generate similarity functions to identify
record replicas in a given repository. The generated
similarity functions properly combine and weight the
best evidence available among the record fields in
order to tell when two distinct records represent the
same real-world entity. The results of the experiments
show that our approach outperforms the baseline method
by Fellegi and Sunter by more than 12percent when
identifying replicas in a data set containing
researcher's personal data, and by more than 7percent,
in a data set with article citation data},
added-at = {2008-06-19T17:35:00.000+0200},
address = {Chapel Hill, NC, USA},
author = {{de Carvalho}, Moises G. and Goncalves, Marcos Andre and Laender, Alberto H. F. and {da Silva}, Altigran S.},
biburl = {https://www.bibsonomy.org/bibtex/26db67a1ce4a28c119dffdf2f7c326f42/brazovayeye},
booktitle = {Proceedings of the 6th ACM/IEEE-CS Joint Conference on
Digital Libraries, JCDL '06},
doi = {doi:10.1145/1141753.1141760},
interhash = {d50bd96f6d82276b4ebba377b62f3c2b},
intrahash = {6db67a1ce4a28c119dffdf2f7c326f42},
isbn = {1-59593-354-9},
keywords = {Deduplication, Digital Libraries algorithms, genetic programming,},
month = {June},
notes = {Comput. Sci. Dept., Fed. Univ. of Minas Gerais, Belo
Horizonte},
pages = {41--50},
publisher = {IEEE},
size = {10 pages},
timestamp = {2008-06-19T17:38:28.000+0200},
title = {Learning to deduplicate},
url = {http://delivery.acm.org/10.1145/1150000/1141760/p41-decarvalho.pdf?key1=1141760&key2=6906456911&coll=GUIDE&dl=GUIDE&CFID=45325455&CFTOKEN=75817203},
year = 2006
}