function [f,g] = logistic_regression(theta, X,y)  %  % Arguments:  %   theta - A column vector containing the parameter values to optimize.  %   X - The examples stored in a matrix.    %       X(i,j) is the i‘th coordinate of the j‘th example.  %   y - The label for each example.  y(j) is the j‘th example‘s label.  %  m=size(X,2);    % initialize objective value and gradient.  f = 0;  g = zeros(size(theta));%%% YOUR CODE HERE %%%  h = 1./(1+exp(-theta‘*X));  f = -y*log(h‘)+(y-1)*log(1-h‘);  g = X*(h-y)‘;

function [f,g] = softmax_regression(theta, X,y)  %  % Arguments:  %   theta - A vector containing the parameter values to optimize.  %       In minFunc, theta is reshaped to a long vector.  So we need to  %       resize it to an n-by-(num_classes-1) matrix.  %       Recall that we assume theta(:,num_classes) = 0.  %  %   X - The examples stored in a matrix.    %       X(i,j) is the i‘th coordinate of the j‘th example.  %   y - The label for each example.  y(j) is the j‘th example‘s label.  %  m=size(X,2);  n=size(X,1);  % theta is a vector;  need to reshape to n x num_classes.  theta=reshape(theta, n, []);  num_classes=size(theta,2)+1;    % initialize objective value and gradient.  %  % TODO:  Compute the softmax objective function and gradient using vectorized code.  %        Store the objective function value in ‘f‘, and the gradient in ‘g‘.  %        Before returning g, make sure you form it back into a vector with g=g(:);  %%%% YOUR CODE HERE %%%    f = 0;  g = zeros(size(theta));  a = theta‘*X;  a = [a;zeros(1,size(a,2))];  a = exp(a);  aSum = sum(a);  compareMatrix = 1:10;  compareMatrix = repmat(compareMatrix‘,1,m);  h=log(a./repmat(aSum,num_classes,1));  judMatrix = abs(compareMatrix - repmat(y,num_classes,1));  A = judMatrix;  A(judMatrix>0) = 0;  A(judMatrix==0) = 1;  B = A*h‘;  f = -sum(diag(B));  g = -X*(A-a./repmat(aSum,num_classes,1))‘;  g = g(:,1:9);  g=g(:); % make gradient a vector for minFunc

%%================================================================%% Step 0a: Load data%  Here we provide the code to load natural image data into x.%  x will be a 784 * 600000 matrix, where the kth column x(:, k) corresponds to%  the raw image data from the kth 12x12 image patch sampled.%  You do not need to change the code below.clear all;close all;clc;x = loadMNISTImages(‘train-images-idx3-ubyte‘);figure(‘name‘,‘Raw images‘);randsel = randi(size(x,2),200,1); % A random selection of samples for visualizationdisplay_network(x(:,randsel));%%================================================================%% Step 0b: Zero-mean the data (by row)%  You can make use of the mean and repmat/bsxfun functions.%%% YOUR CODE HERE %%%xMeanRow = mean(x);x = x-repmat(xMeanRow,size(x,1),1);%%================================================================%% Step 1a: Implement PCA to obtain xRot%  Implement PCA to obtain xRot, the matrix in which the data is expressed%  with respect to the eigenbasis of sigma, which is the matrix U.%%% YOUR CODE HERE %%%xCorr = x*x‘/size(x,2);[U S V] = svd(xCorr);xRot = U‘*x;%%================================================================%% Step 1b: Check your implementation of PCA%  The covariance matrix for the data expressed with respect to the basis U%  should be a diagonal matrix with non-zero entries only along the main%  diagonal. We will verify this here.%  Write code to compute the covariance matrix, covar. %  When visualised as an image, you should see a straight line across the%  diagonal (non-zero entries) against a blue background (zero entries).%%% YOUR CODE HERE %%%covar = xRot*xRot‘/size(xRot,2);% Visualise the covariance matrix. You should see a line across the% diagonal against a blue background.figure(‘name‘,‘Visualisation of covariance matrix‘);imagesc(covar);%%================================================================%% Step 2: Find k, the number of components to retain%  Write code to determine k, the number of components to retain in order%  to retain at least 99% of the variance.%%% YOUR CODE HERE %%%var = sum(diag(covar));varMin = 0.99*var;varSum = 0;k = 0;A = diag(covar);for i=1:length(A)    varSum = varSum+A(i);    if(varSum>=varMin && k==0)        k = i;    endend%%================================================================%% Step 3: Implement PCA with dimension reduction%  Now that you have found k, you can reduce the dimension of the data by%  discarding the remaining dimensions. In this way, you can represent the%  data in k dimensions instead of the original 144, which will save you%  computational time when running learning algorithms on the reduced%  representation.% %  Following the dimension reduction, invert the PCA transformation to produce %  the matrix xHat, the dimension-reduced data with respect to the original basis.%  Visualise the data and compare it to the raw data. You will observe that%  there is little loss due to throwing away the principal components that%  correspond to dimensions with low variation.%%% YOUR CODE HERE %%%xRot = U‘*x;xTilde = U(:,1:k)‘*x;xHat = U*[xTilde;zeros(size(x,1)-k,size(x,2))];% Visualise the data, and compare it to the raw data% You should observe that the raw and processed data are of comparable quality.% For comparison, you may wish to generate a PCA reduced image which% retains only 90% of the variance.figure(‘name‘,[‘PCA processed images ‘,sprintf(‘(%d / %d dimensions)‘, k, size(x, 1)),‘‘]);display_network(xHat(:,randsel));figure(‘name‘,‘Raw images‘);display_network(x(:,randsel));%%================================================================%% Step 4a: Implement PCA with whitening and regularisation%  Implement PCA with whitening and regularisation to produce the matrix%  xPCAWhite. epsilon = 1e-1; %%% YOUR CODE HERE %%%xPCAWhite = diag(1./sqrt(diag(S) + epsilon)) * xRot;covar = xPCAWhite*xPCAWhite‘;%% Step 4b: Check your implementation of PCA whitening %  Check your implementation of PCA whitening with and without regularisation. %  PCA whitening without regularisation results a covariance matrix %  that is equal to the identity matrix. PCA whitening with regularisation%  results in a covariance matrix with diagonal entries starting close to %  1 and gradually becoming smaller. We will verify these properties here.%  Write code to compute the covariance matrix, covar. %%  Without regularisation (set epsilon to 0 or close to 0), %  when visualised as an image, you should see a red line across the%  diagonal (one entries) against a blue background (zero entries).%  With regularisation, you should see a red line that slowly turns%  blue across the diagonal, corresponding to the one entries slowly%  becoming smaller.%%% YOUR CODE HERE %%%% Visualise the covariance matrix. You should see a red line across the% diagonal against a blue background.figure(‘name‘,‘Visualisation of covariance matrix‘);imagesc(covar);%%================================================================%% Step 5: Implement ZCA whitening%  Now implement ZCA whitening to produce the matrix xZCAWhite. %  Visualise the data and compare it to the raw data. You should observe%  that whitening results in, among other things, enhanced edges.%%% YOUR CODE HERE %%%xZCAWhite = U * diag(1./sqrt(diag(S) + epsilon)) * U‘ * x;% Visualise the data, and compare it to the raw data.% You should observe that the whitened images have enhanced edges.figure(‘name‘,‘ZCA whitened images‘);display_network(xZCAWhite(:,randsel));figure(‘name‘,‘Raw images‘);display_network(x(:,randsel));