ID3算法的java实现
ID3算法是经典的决策树学习生成算法。ID3算法的核心是在决策树各个节点上运用信息增益准则选择特征，递归的构建决策树。具体方法是：从根节点（root node）开始，对接点计算所有可能的特征的信息增益，选择信息增益最大的特征作为节点的特征，有该特征的不同取值建立子节点；再对子节点递归的调用以上方法，构建决策树；直到所有的特征的信息增益均很小或者没有特征可以选取为止。最后得到一个决策树。要理解ID3算法，需要先了解一些基本的信息论概念，包括信息量，熵，后验熵，条件熵。
java代码
/** * C4.5决策树数据结构 * @author zhenhua.chen * @Description: TODO * @date 2013-3-1 上午10:47:37 * */
public class TreeNode {
private String nodeName; // 决策树节点名称
private List<String> splitAttributes; // 分裂属性名
private ArrayList<TreeNode> childrenNodes; // 决策树的子节点
private ArrayList<ArrayList<String>> dataSet; // 划分到该节点的数据集
private ArrayList<String> arrributeSet; // 数据集所有属性
public TreeNode(){
childrenNodes = new ArrayList<TreeNode>();
}
public String getNodeName() {
return nodeName;
}
public void setNodeName(String nodeName) {
this.nodeName = nodeName;
}
public List<String> getSplitAttributes() {
return splitAttributes;
}
public void setSplitAttributes(List<String> splitAttributes) {
this.splitAttributes = splitAttributes;
}
public ArrayList<TreeNode> getChildrenNodes() {
return childrenNodes;
}
public void setChildrenNodes(ArrayList<TreeNode> childrenNodes) {
this.childrenNodes = childrenNodes;
}
public ArrayList<ArrayList<String>> getDataSet() {
return dataSet;
}
public void setDataSet(ArrayList<ArrayList<String>> dataSet) {
this.dataSet = dataSet;
}
public ArrayList<String> getArrributeSet() {
return arrributeSet;
}
public void setArrributeSet(ArrayList<String> arrributeSet) {
this.arrributeSet = arrributeSet;
}
} 决策树算法：
/** * 构造决策树的类 * @author zhenhua.chen * @Description: TODO * @date 2013-3-1 下午4:42:07 * */
public class DecisionTree {
/** * 建树类 * @param dataSet * @param attributeSet * @return */
public TreeNode buildTree(ArrayList<ArrayList<String>> dataSet, ArrayList<String> attributeSet) {
TreeNode node = new TreeNode();
node.setDataSet(dataSet);
node.setArrributeSet(attributeSet);
// 根据当前数据集计算决策树的节点
int index = -1;
double gain = 0;
double maxGain = 0;
for(int i = 0; i < attributeSet.size() - 1; i++) {
gain = ComputeUtil.computeEntropy(dataSet, attributeSet.size() - 1) - ComputeUtil.computeConditinalEntropy(dataSet, i);
if(gain > maxGain) {
index = i;
maxGain = gain;
}
}
ArrayList<String> splitAttributes = ComputeUtil.getTypes(dataSet, index); // 获取该节点下的分裂属性
node.setSplitAttributes(splitAttributes);
node.setNodeName(attributeSet.get(index));
// 判断每个属性列是否需要继续分裂
for(int i = 0; i < splitAttributes.size(); i++) {
ArrayList<ArrayList<String>> splitDataSet = ComputeUtil.getDataSet(dataSet, index, splitAttributes.get(i));
// 判断这个分裂子数据集的目标属性是否纯净，如果纯净则结束，否则继续分裂
int desColumn = splitDataSet.get(0).size() - 1; // 目标属性列所在的列号
ArrayList<String> desAttributes = ComputeUtil.getTypes(splitDataSet, desColumn);
TreeNode childNode = new TreeNode();
if(desAttributes.size() == 1) {
childNode.setNodeName(desAttributes.get(0));
} else {
ArrayList<String> newAttributeSet = new ArrayList<String>();
for(String s : attributeSet) { // 删除新属性集合中已作为决策树节点的属性值
if(!s.equals(attributeSet.get(index))) {
newAttributeSet.add(s);
}
}
ArrayList<ArrayList<String>> newDataSet = new ArrayList<ArrayList<String>>();
for(ArrayList<String> data : splitDataSet) { // 除掉columnIndex参数指定的
ArrayList<String> tmp = new ArrayList<String>();
for(int j = 0; j < data.size(); j++) {
if(j != index) {
tmp.add(data.get(j));
}
}
newDataSet.add(tmp);
}
childNode = buildTree(newDataSet, newAttributeSet); // 递归建树
}
node.getChildrenNodes().add(childNode);
}
return node;
}
/** * 打印建好的树 * @param root */
public void printTree(TreeNode root) {
System.out.println("----------------");
if(null != root.getSplitAttributes()) {
System.out.print("分裂节点：" + root.getNodeName());
for(String attr : root.getSplitAttributes()) {
System.out.print("(" + attr + ") ");
}
} else {
System.out.print("分裂节点：" + root.getNodeName());
}
if(null != root.getChildrenNodes()) {
for(TreeNode node : root.getChildrenNodes()) {
printTree(node);
}
}
}
/** * * @Title: searchTree * @Description: 层次遍历树 * @return void * @throws */
public void searchTree(TreeNode root) {
Queue<TreeNode> queue = new LinkedList<TreeNode>();
queue.offer(root);
while(queue.size() != 0) {
TreeNode node = queue.poll();
if(null != node.getSplitAttributes()) {
System.out.print("分裂节点：" + node.getNodeName() + "; ");
for(String attr : node.getSplitAttributes()) {
System.out.print(" (" + attr + ") ");
}
} else {
System.out.print("叶子节点：" + node.getNodeName() + "; ");
}
if(null != node.getChildrenNodes()) {
for(TreeNode nod : node.getChildrenNodes()) {
queue.offer(nod);
}
}
}
}
} 一些util代码：
/** * C4.5算法所需的各类计算方法 * @author zhenhua.chen * @Description: TODO * @date 2013-3-1 上午10:48:47 * */
public class ComputeUtil {
/** * 获取指定数据集中指定属性列的各个类别 * @Title: getTypes * @Description: TODO * @return ArrayList<String> * @throws */
public static ArrayList<String> getTypes(ArrayList<ArrayList<String>> dataSet, int columnIndex) {
ArrayList<String> list = new ArrayList<String>();
for(ArrayList<String> data : dataSet) {
if(!list.contains(data.get(columnIndex))) {
list.add(data.get(columnIndex));
}
}
return list;
}
/** * 获取指定数据集中指定属性列的各个类别及其计数 * @Title: getClassCounts * @Description: TODO * @return Map<String,Integer> * @throws */
public static Map<String, Integer> getTypeCounts(ArrayList<ArrayList<String>> dataSet, int columnIndex) {
Map<String, Integer> map = new HashMap<String, Integer>();
for(ArrayList<String> data : dataSet) {
String key = data.get(columnIndex);
if(map.containsKey(key)) {
map.put(key, map.get(key) + 1);
} else {
map.put(key, 1);
}
}
return map;
}
/** * 获取指定列上指定类别的数据集合(分裂后的数据子集) * @Title: getDataSet * @Description: TODO * @return ArrayList<ArrayList<String>> * @throws */
public static ArrayList<ArrayList<String>> getDataSet(ArrayList<ArrayList<String>> dataSet, int columnIndex, String attribueClass) {
ArrayList<ArrayList<String>> splitDataSet = new ArrayList<ArrayList<String>>();
for(ArrayList<String> data : dataSet) {
if(data.get(columnIndex).equals(attribueClass)) {
splitDataSet.add(data);
}
}
return splitDataSet;
}
/** * 计算指定列(属性)的信息熵 * @Title: computeEntropy * @Description: TODO * @return double * @throws */
public static double computeEntropy(ArrayList<ArrayList<String>> dataSet, int columnIndex) {
Map<String, Integer> map = getTypeCounts(dataSet, columnIndex);
int dataSetSize = dataSet.size();
Iterator<String> keyIter = map.keySet().iterator();
double entropy = 0;
while(keyIter.hasNext()) {
double prob = (double)map.get((String)keyIter.next()) / (double)dataSetSize;
entropy += (-1) * prob * Math.log(prob) / Math.log(2);
}
return entropy;
}
/** * 计算基于指定属性列对目标属性的条件信息熵 */
public static double computeConditinalEntropy(ArrayList<ArrayList<String>> dataSet, int columnIndex) {
Map<String, Integer> map = getTypeCounts(dataSet, columnIndex); // 获取该属性列的所有列别及其计数
double conditionalEntropy = 0; // 条件熵
// 获取根据每个类别分割后的数据集合
Iterator<String> iter = map.keySet().iterator();
while(iter.hasNext()) {
ArrayList<ArrayList<String>> splitDataSet = getDataSet(dataSet, columnIndex, (String)iter.next());
// 计算目标属性列的列索引
int desColumn = 0;
if(splitDataSet.get(0).size() > 0) {
desColumn = splitDataSet.get(0).size() - 1;
}
double probY = (double)splitDataSet.size() / (double)dataSet.size();
Map<String, Integer> map1 = getTypeCounts(splitDataSet, desColumn); //根据分割后的子集计算后验熵
Iterator<String> iter1 = map1.keySet().iterator();
double proteriorEntropy = 0;
while(iter1.hasNext()) {
String key = (String)iter1.next(); // 目标属性列中的一个分类
double posteriorProb = (double)map1.get(key) / (double)splitDataSet.size();
proteriorEntropy += (-1) * posteriorProb * Math.log(posteriorProb) / Math.log(2);
}
conditionalEntropy += probY * proteriorEntropy; // 基于某个分割属性计算条件熵
}
return conditionalEntropy;
}
}测试代码：
public class Test {
public static void main(String[] args) {
File f = new File("D:/test.txt");
BufferedReader reader = null;
try {
reader = new BufferedReader(new FileReader(f));
String str = null;
try {
str = reader.readLine();
ArrayList<String> attributeList = new ArrayList<String>();
String[] attributes = str.split("\t");
for(int i = 0; i < attributes.length; i++) {
attributeList.add(attributes[i]);
}
ArrayList<ArrayList<String>> dataSet = new ArrayList<ArrayList<String>>();
while((str = reader.readLine()) != null) {
ArrayList<String> tmpList = new ArrayList<String>();
String[] s = str.split("\t");
for(int i = 0; i < s.length; i++) {
tmpList.add(s[i]);
}
dataSet.add(tmpList);
}
DecisionTree dt = new DecisionTree();
TreeNode root = dt.buildTree(dataSet, attributeList);
// dt.printTree(root);
dt.searchTree(root);
} catch (IOException e) {
e.printStackTrace();
}
} catch (FileNotFoundException e) {
e.printStackTrace();
}
}
} 