#####Chapter 5: Matrix Decompositions

###Section 5.1:  Eigen Decomposition
#Matrix multiplication with an ordinary vector (b)
A <- matrix(c(5,6,1,4), nrow=2, ncol=2, byrow=TRUE)  
b <- matrix(c(2,6), nrow=2, ncol=1, byrow=TRUE)  
scalar=7

mult.1 <-b*scalar; mult.1
mult.2 <-A%*%b; mult.2

#Matrix multiplication with an eigenvector (c)
c <- matrix(c(6,2), nrow=2, ncol=1, byrow=TRUE)
mult.3 <-c*scalar; mult.3 
mult.4 <-A%*%c; mult.4

#Algebraic function for finding eigenpairs from a 2X2 matrix
eigs <-function(A){
a <-1
b <--(A[1,1]+A[2,2])
c <-(A[1,1]*A[2,2])-(A[1,2]*A[2,1])
lambda1 <-(-b+sqrt(b^2-4*c))/(2*a)
lambda2 <-(-b-sqrt(b^2-4*c))/(2*a)
v1 <-(-A[1,2])/(A[1,1]-lambda1)
vec1 <-matrix(cbind(v1,1)/(sqrt(sum(v1^2+1^2))),nrow=2)
v2 <-(-(A[1,1]-lambda2))/(A[1,2])
vec2 <-matrix(cbind(1,v2)/(sqrt(sum(1^2+v2^2))),nrow=2)
out <-matrix(c(lambda1,vec1,lambda2,vec2),nrow=3,ncol=2)
}

A <-matrix(c(5,6,1,4),nrow=2,ncol=2,byrow=T);A
eigpairs <-eigs(A)
dimnames(eigpairs)=list(c("lambda","v1","v2"),c("pair1","pair2"))
eigpairs

#Reconstruct A
lambda <-diag(c(eigpairs[1],eigpairs[4]))
V <-matrix(c(eigpairs[2],eigpairs[3],eigpairs[5],eigpairs[6]),nrow=2,ncol=2)
reconstruct <-V%*%lambda%*%solve(V)
reconstruct


###Section 5.2: QR Decomposition and QR Algorithm
x1 <-c(1,9,1,5,6,8,2,4,2,8,7,7)
x2 <-c(2,7,1,8,5,7,4,7,1,4,3,4)
X <-cbind(1,x1,x2)
y <-c(3,8,2,8,5,9,4,5,2,4,2,6)

qr.house=function(A){
    m=nrow(A)
    n=ncol(A)
    B=A
    I=diag(1,m)
    Q=diag(1,m)
    R=diag(0,n)
    for (k in 1:n){
        v=matrix(c(rep(0,m)),byrow=T)
        v[k:m]=B[k:m,k]
        i=matrix(c(rep(0,m)),byrow=T)
        i[k]=1
        u = sign(v[k])*norm(v,type="2")*i + v
        u = u/norm(u,type="2")
        Hk = I - 2*u%*%t(u)
        R=Hk%*%B
        Q=Q%*%Hk
        B=R
    }
    RQ <- list("R" = round(R[1:k,1:k],15), "Q" = Q[1:m,1:k]);RQ
}
qr.house(X)
A= qr.house(X)$Q%*%qr.house(X)$R;A 

#QR Regression using Rs QR function
R.QR <-qr(X)
R <-qr.R(R.QR)
Q <-qr.Q(R.QR)
QR.beta <-solve(R)%*%t(Q)%*%y;QR.beta
QR.fit <-Q%*%t(Q)%*%y;QR.fit
df <-length(y)-ncol(X)
MSe <-(sum((y-QR.fit)^2)/df)
covar <-MSe*solve(R)%*%t(solve(R));covar 

#QR Algorithm for Finding Eigen pairs (default method in R)
XX <-t(X)%*%X
eigs <-eigen(XX);eigs

###Section 5.3: Singular Value Decomposition
#SVD for Regression
x0 <-rep(1,12)
x1 <-c(1,9,1,5,6,8,2,4,2,8,7,7)
x2 <-c(2,7,1,8,5,7,4,7,1,4,3,4)
y <-c(3,8,2,8,5,9,4,5,2,4,2,6)
X <-cbind(x0,x1,x2)
svd.reg <-svd(X)
svd.reg
D <-diag(svd.reg$d)
V <-svd.reg$v
U <-svd.reg$u

#Reconstruct X
X_rc<-U%*%D%*%solve(V)
X_rc

#Regression Coefficients
svd.beta <-V%*%solve(D)%*%t(U)%*%y
svd.beta

#Standard Errors Using Eigen Pairs
V1 <-V[1, ]
c1 <-sum(V1^2/svd.reg$d^2)
V2 <-V[2, ]
c2 <-sum(V2^2/svd.reg$d^2)
V3 <-V[3, ]
c3 <-sum(V3^2/svd.reg$d^2)
C <-cbind(c1,c2,c3)
reg <-lm(y~x1+x2)
mse <-summary(reg)$sigma^2

std.errors <- sqrt(mse*C)
std.errors

###Section 5.4:  Cholesky Decomposition
x0 <-rep(1,12)
x1 <-c(1,9,1,5,6,8,2,4,2,8,7,7)
x2 <-c(2,7,1,8,5,7,4,7,1,4,3,4)
y <-c(3,8,2,8,5,9,4,5,2,4,2,6)
X <-cbind(x0,x1,x2)
XX <-t(X)%*%X

M=XX 
n = max(dim(M)) 
L = matrix(0, nrow=n, ncol=n) 
for (i in 1:n) {
   L[i,i] = sqrt(M[i,i] - L[i,,drop=FALSE] %*% t(L[i,,drop=FALSE]))
   if (i < n) {
     for (j in (i+1):n) {
       L[j,i] = (M[j,i] - L[i,,drop=FALSE] %*% t(L[j,,drop=FALSE]))/L[i,i]
     }
   }
}
L

#Calculate Determinant
determ <-prod(diag(L)^2)
determ

#Calculate Inverse
XXinv <-t(solve(L))%*%solve(L)
XXinv

#Cholesky Regression
chol.beta <-solve(t(L))%*%t(X%*%solve(t(L)))%*%y
chol.beta
chol.fitted <-X%*%solve(t(L))%*%t(X%*%solve(t(L)))%*%y
chol.fitted
covar <- t(solve(L))%*%solve(L)*sum((y-chol.fitted)^2)/9
covar

#Cholesky Eigenvalues (10 iterations)
CC <- t(X)%*%X;
for(i in 1:10)
{
  CC <- chol(CC);
  CC <-CC%*%t(CC);
}
lambda <-diag(diag(CC))
lambda
