%
% wrspec.tex
%
% White Rabbit Spec. Currently an informal draft.
%
% best master clock algorithm 6.6.2.3
% domain number, profiles 7.1, annex J (p. 237)
% asymmetry 6.2.c, 7.4.2 (positive if t_MS > t_SM)
% 
% TODO
% Give the values of different constants in tables.
% Use thicker line for combined state in wr state machine figure.
% Take cursor out of PTP state machine figure.

\documentclass[a4paper, 12pt]{article}
%\documentclass{article}

\usepackage{fullpage}
\usepackage{pgf}
\usepackage{tikz}
\usetikzlibrary{arrows,automata,shapes}
\usepackage{multirow}
\usepackage{color}
\usepackage[latin1]{inputenc}
\usepackage{verbatim}
\usepackage{amsmath}
\usepackage{times,mathptmx}
\usepackage{chngcntr}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% creating subsubsubsection notation
% src: http://www.latex-community.org/forum/viewtopic.php?f=5&t=791
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\setcounter{secnumdepth}{6}
\renewcommand\theparagraph{\Alph{paragraph}}

\makeatletter
\renewcommand\paragraph{\@startsection{paragraph}{4}{\z@}%
                                     {-3.25ex\@plus -1ex \@minus -.2ex}%
                                     {0.0001pt \@plus .2ex}%
                                     {\normalfont\normalsize\bfseries}}
\renewcommand\subparagraph{\@startsection{subparagraph}{5}{\z@}%
                                     {-3.25ex\@plus -1ex \@minus -.2ex}%
                                     {0.0001pt \@plus .2ex}%
                                     {\normalfont\normalsize\bfseries}}

\counterwithin{paragraph}{subsubsection}
\counterwithin{subparagraph}{paragraph}
\makeatother
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\newcommand{\eqoffset}[1]{%
  {\ensuremath{%
      {\text{offset}}_{#1}}%
  }%
}
\newcommand{\eqdelay}[1]{{\text{delay}}_{#1}}
\newcommand{\eqasymm}{{\text{asymmetry}}}

\begin{document}

\title{White Rabbit Specification: \\Draft for Comments}
\author{Emilio G. Cota\\Maciej Lipinski\\Tomasz W\l{}ostowski\\Erik van der Bij\\Javier Serrano}
\date{September 2010}
\maketitle
\thispagestyle{empty}

\begin{figure}[ht!]
  \centering
  \vspace{1.3cm}
  \includegraphics[width=0.50\textwidth]{fig/wr_logo.ps}
  \label{fig:wr_logo}
\end{figure}

\newpage

\setcounter{page}{1}
\begin{center}
\large Revision History Table 
\end{center} 


\begin{table}[ht]
%\caption{White Rabbit Specification Revision History Table } 
\centering
\begin{tabular}{| c |  c | c | p{8cm}    |}          \hline
\textbf{Version} & \textbf{Date} & \textbf{Authors} & \textbf{Description}  \\ 
 & &  &\\ \hline
0.1 & 10/09/2010 & E.G., M.L.   & First draft for comments.  \\ \hline
0.2 & 7/09/2010  & T.W., M.L.   & Added introduction about PTP.  \\ \hline
0.3 & 8/09/2010  & J.S., M.L.   & Language\&Style-related corrections, reference to Peter's paper.  
\\ \hline
0.4 & 14/09/2010 & E.V.D.B., M.L.& 1.~Merged the FSM of the Slave and the FSM of the Master into 
                                       single and linear FSM,\\
    &            &              & 2.~Changed WR managementIDs,\\
    &            &              & 3.~Added authors, this rev. table, explanatory figures and 
                                                              descriptions. \\ \hline
0.5 & 17/09/2010 & E.V.D.B., J.S.& 1.~Added Table of Contents. \\
    &            &               & 2.~Re-ordered appendixes. \\
    &            &               & 3.~Fixed some typos. \\ 
\hline
1.0 & 26/04/2011 & M.L.          & 1.~Including feedback from comments. \\
    &            &               & 2.~Creating WR PTP Profile. \\
    &            &               & 3.~Adding modification to BMC. \\ 
    &            &               & 4.~Enabling WR Switch to be "real" Boundary Clock. \\ 
    &            &               & 5.~Adding figure clarifying WRPTP vs. PTP FSMs operation from 
                                      Power On. \\ 
    &            &               & 6.~Mentioning (missed) modification to PTP FSM. \\ 
\hline
\end{tabular}
\label{tab:revHist}
\end{table}

\newpage

\tableofcontents

\newpage

\section{Introduction}

White Rabbit (WR) is a protocol developed to synchronize nodes in 
a packet-based network with sub-ns accuracy. The protocol results 
from the combination of IEEE1588-2008 (PTP)\cite{IEEE1588} with two further
requirements: precise knowledge of the link delay and clock
syntonization\footnote{The adjustment of two electronic circuits or devices in terms of frequency.} 
over the physical layer.

A WR link is formed by a pair of nodes, master and slave (Figure~\ref{fig:wrNodes}). The master
node uses a traceable clock to encode data over the physical layer, while the slave
recovers this clock (syntonization) and bases its timekeeping on it. Absolute time
synchronisation between master and slave is achieved by adjusting
the clock phase and offset of the slave to that of the master. 
The \textit{offset} refers to the clock (e.g. time defined in Coordinated Universal Time standard), 
while the \textit{phase} refers to the clock signal (e.g. 125 MHz clock signal). 
The phase and offset adjustment is done through the two-way exchange of PTP sync messages, which 
are corrected
to achieve sub-ns accuracy due to the precise knowledge of the link delay.

\begin{figure}[ht!]
  \centering
%  \vspace{-1.3cm}
  \includegraphics[width=1.00\textwidth]{fig/clocks.ps}
  \caption{Synchronization and syntonization in a White Rabbit link.}
  \label{fig:wrNodes}
\end{figure}


\newpage

The precise knowledge of the link delay is obtained by accurate hardware 
timestamps and calculation of the delay asymmetry (see section~\ref{sec:delayAsymCal}).

The described single-link synchronization can be replicated. 
Multi-link WR networks are obtained by chaining WR links forming a
hierarchical topology. This hierarchy is imposed by the fact that
a frequency traceable to a common grandmaster must be distributed
over the physical layer, resulting in a \textit{cascade} of master
and slave nodes. As a result of this topology, a WR~network consists of two kinds of WR network 
devices:
\textit{WR boundary clocks} (WR Switches) and \textit{WR ordinary clocks} 
(WR Nodes), see section~\ref{sec:wrptpOverview}.

\begin{figure}[ht!]
  \centering
%  \vspace{-1.3cm}
  \includegraphics[width=0.80\textwidth]{fig/wr_net_topology.ps}
  \caption{White Rabbit network; it forms a hierarchical topology.}
  \label{fig:wrNetwork}
\end{figure}

Applications need WR and IEEE1588-2008 nodes to coexist. Examples
of this are networks where the need for highly accurate time synchronisation is concentrated on a
certain group of nodes. For this purpose the WR protocol enables WR nodes to defer to IEEE1588
behaviour when not connected to another WR node.




\newpage

\section{Precision Time Protocol}
\label{sec:ptp}

The IEEE1588-2008 standard, known as Precision Time Protocol (PTP), 
is repeatedly referenced in this document. Knowledge of basic PTP concepts is 
required to read this specification, therefore they are explained in this section. 

PTP is a packet-based protocol designed to synchronize devices in distributed systems. 
The standard defines two kinds of messages which are exchanged between \textit{PTP nodes}: 
\textit{event messages} and \textit{general messages}. Both, the time of transmission and 
the time of reception of event messages are \textit{timestamped}. General messages 
are used by PTP nodes to identify other PTP nodes, establish clock hierarchy and 
exchange data, e.g.~timestamps, settings or parameters. PTP defines several methods 
for node synchronization. Figure~\ref{fig:wrPTPmsgs} presents the messages used when 
the \textit{delay request-response mechanism} (with a \textit{two-step clock}) is used, 
which is the case in White Rabbit. For simplicity reasons, a PTP node is 
considered an \textit{ordinary clock} in the remainder of this section; such clocks 
have only one port. 

\begin{figure}[ht!]
  \centering
  \includegraphics[width=0.30\textwidth]{fig/ptpMSGs.ps}
  \caption{PTP messages used by WRPTP.}
  \label{fig:wrPTPmsgs}
\end{figure}

An \textit{Announce Message} is periodically broadcast by the PTP node which is 
in the Master state. The message carries information about its originator and 
the originator's clock source quality. This enables other PTP nodes receiving 
the announce message to perform the Best Master Clock (BMC) Algorithm. 
This algorithm defines the role of each PTP node in the PTP network hierarchy; 
the outcome of the algorithm is the recommended next state of the PTP node and 
the node's synchronization source (grandmaster). In other words, a PTP node 
decides to which other PTP node it should synchronize based on the information 
provided in the announce messages and using the BMC algorithm. A PTP node which is in 
the SLAVE state synchronizes to the clock of another PTP node. A PTP node 
which is in the MASTER state is regarded as a source of synchronization 
for the other PTP nodes. The full PTP state machine with state descriptions 
is included in Appendix~\ref{ptpFSM}.



\textit{Sync Messages} and \textit{Delay\_Req Messages} are timestamped 
($t_{1}$, $t_{2}$, $t_{3}$, $t_{4}$) and these timestamps are used to calculate 
the offset and the delay between the nodes exchanging the messages. \textit{Follow\_UP Messages} 
and \textit{Delay\_Resp Messages} are used to send timestamps between Master 
and Slave (in the case of a two-step clock). 

\textit{Management Messages} are used only for configuration and 
administrative purposes. They are not essential for PTP synchronization.

The flow of events in the PTP delay request-response (two-step clock) mechanism is the following 
(simplified overview): 
\begin{enumerate}
\item The master sends Announce messages periodically.
\item The slave receives the Announce message and uses the BMC algorithm to establish its place in 
      the network hierarchy.
\item The master periodically sends a Sync message (timestamped on transmission, $t_1$) followed by 
      a Follow\_UP message which carries $t_1$.
\item The slave receives the Sync message sent by the master (timestamped on reception, $t_2$).
\item The slave receives the Follow\_Up message (which carries timestamp of Sync transmission time)
      sent by the master .
\item The slave sends a Delay\_Req message (timestamped on transmission, $t_3$).
\item The master receives the Delay\_Req message sent by the slave (timestamped on reception, 
      $t_4$).
\item The master sends the Delay\_Resp message which carries $t_4$.
\item The slave receives the Delay\_Resp.
\item The slave adjusts its clock using the offset and the delay calculated with timestamps 
      ($t_{1}$, $t_{2}$, $t_{3}$, $t_{4}$). It results in the Slave's synchronization with the
      Master clock.
\item Repeat 1-10.
\end{enumerate}


\newpage

\section{Link Delay Model}
\label{sec:delaymodel}

The delay of a message travelling from master to slave (see Figure~\ref{fig:delaymodel}) 
can be expressed as the sum

\begin{equation}
  \label{eq:delayms}
  \eqdelay{ms} = \Delta_{tx_m} + \delta_{ms} + \Delta_{rx_s}
\end{equation}

where $\Delta_{tx_m}$ is the fixed delay due to the master's transmission
circuitry, $\delta_{ms}$ is the variable delay incurred in the
transmission medium and $\Delta_{rx_s}$ is the fixed delay due to the
slave's reception circuitry.
In a similar fashion, the delay of a message travelling from slave to
master can be decomposed as

\begin{equation}
  \label{eq:delaysm}
  \eqdelay{sm} = \Delta_{tx_s} + \delta_{sm} + \Delta_{rx_m}
\end{equation}

The characterization of the link is completed with an equation to
relate the two variable delays $\delta_{ms}$ and $\delta_{sm}$.
%  From now
% on in this document we refer to this missing equation as the \textit{physical
% medium correlation}. 
Describing a procedure to obtain this equation is out of the scope of this document. However, 
section~\ref{sec:physcorr} provides such equation obtained empirically for one scenario.

\begin{figure}[ht!]
  \centering
    \includegraphics[width=0.95\textwidth]{fig/delaymodel.eps}
  \caption{Delay model of a WR link. The timestamps are accurately corrected
    for link asymmetries by the usage of the four fixed delays
    $\Delta_{\{tx_m, rx_s, tx_s, rx_m\}}$ and the relationship
    between both variable delays $\delta_{\{ms, sm\}}$. }
  \label{fig:delaymodel}
\end{figure}

\subsection{Relative Delay Coefficient}
\label{sec:physcorr}

An accurate relation between both variable delays on the transmission line
is essential for obtaining an acceptable estimate of the delay asymmetry on a
WR link. The relation between $\delta_{ms}$ and $\delta_{sm}$ is represented in 
this document by the \textit{relative delay coefficient} ($\alpha$).
Its origin is highly implementation-dependent. Thus this document just assumes that it exists 
and is known. 

\subsubsection{Ethernet over a Single-mode Optical Fibre}
\label{sec:singlefiber}
When a single-mode fibre is used as bi-directional communication medium,
it can be shown that both variable delays are related by an equation of
the form \cite{Peek2010}:
\begin{equation}
\delta_{ms} = (1 + \alpha) \, \delta_{sm}
\label{eq:singlefiber}
\end{equation}

\section{Delay Asymmetry Calculation}
\label{sec:delayAsymCal}

Let us start from the PTP sync timestamps, represented by the familiar
set $t_1$, $t_2$, $t_3$ and $t_4$. The mean path delay is then defined as
\begin{equation}
  \mu = \frac{(t_2 - t_1) + (t_4 - t_3)}{2}
\end{equation}
Note that the transmission delays $t_2 - t_1$ and $t_4 - t_3$ can be
expressed in terms of WR's Delay Model:
\begin{align}
  t_2 - t_1 = \Delta_{tx_m} + \delta_{ms} + \Delta_{rx_s} + \eqoffset{ms}\\
  t_4 - t_3 = \Delta_{tx_s} + \delta_{sm} + \Delta_{rx_m} - \eqoffset{ms}
\end{align} where \eqoffset{ms} is the time offset between the
slave's clock and the master's. Combining the three equations above
we obtain
\begin{equation}
\label{eq:mu}
  2\mu = \Delta + \delta_{sm} + \delta_{ms}
\end{equation} where $\Delta$ accounts for all fixed delays in the path,
i.e.
\begin{equation}
\label{eq:delta}
  \Delta = \Delta_{tx_m} + \Delta_{rx_s} + \Delta_{tx_s} + \Delta_{rx_m}
\end{equation}

The delay asymmetry as specified in section 7.4.2 of the PTP standard is expressed in our own 
notation by using equations \eqref{eq:delayms},
\eqref{eq:delaysm} and \eqref{eq:mu} as follows:
\begin{align}
  \eqdelay{ms} = \mu + \eqasymm
  \label{eq:asymm} \\
  \eqdelay{sm} = \mu - \eqasymm
\end{align}

The delay asymmetry cannot be calculated unless we use the physical medium
correlation.


\subsection{Solution for Ethernet over a Single-mode Optical Fiber}
Combining equations \eqref{eq:singlefiber} and \eqref{eq:mu} we obtain:
\begin{gather}
  \delta_{ms} = \frac{1 + \alpha}{2 + \alpha} \, (2\mu - \Delta)
  \label{eq:deltams} \\
  \delta_{sm} = \frac{2\mu - \Delta}{2 + \alpha}
  \label{eq:deltasm}
\end{gather}
\\
The delay asymmetry can then be derived from equations \eqref{eq:delayms}, \eqref{eq:asymm}, 
\eqref{eq:deltams} and
\eqref{eq:deltasm}:
\begin{equation}
\label{eq:aqasymm}
  \eqasymm = \Delta_{tx_m} + \Delta_{rx_s} - \frac{\Delta - \alpha \mu + \alpha \Delta}{2 + \alpha}
\end{equation}
\\





\section{Fixed delays}
\label{sec:fixedDelays}
The knowledge of fixed delays $\Delta_{\{tx_m, rx_s, tx_s, rx_m\}}$ is necessary to calculate 
delay asymmetry \eqref{eq:aqasymm}. Such delays may be constant for the lifetime of the hardware, 
its up-time or the duration of the link connection. Therefore, the method for obtaining fixed 
delays is medium-specific and implementation-dependent. The delays are measured (if necessary) 
and their values are distributed across the link during the process of 
establishing the WR link, which is called \textit{WR Link Setup} in this document. 
A WR~node participates in the measurement of another 
WR node's reception fixed delay ($\Delta_{\{rx_m, rx_s\}}$) upon request, e.g. by sending 
a calibration pattern in Gigabit Ethernet. Measurement of fixed delays during WR Link Setup 
is optional. It is only required if non-deterministic reception/transmission elements are used.

An example implementation of the method to obtain fixed delays for non-deterministic Gigabit 
Ethernet PHY 
is described in Appendix~\ref{sec:calibForGigbitE}. This fixed delay measurement is optional and in 
principle needed only once, while the WR link is being set up.

\newpage

\section{White Rabbit PTP }


White Rabbit extends the IEEE1588-2008 (PTP) standard to achieve sub-ns accuracy while still 
benefiting from PTP's synchronization, management and messaging mechanisms. From now on in this
document, the White Rabbit extension to the PTP standard will be referred to as \textit{WRPTP}.

WRPTP takes advantage of PTP's customization facilities, i.e. \textit{PTP profile} and 
\textit{Type-Length-Value} (TLV), and makes a few extensions to PTP standard which are out of the 
scope of mentioned facilities. It also defines \textit{implementation specific} functionalities 
(e.g. WR-specific fields of Data Sets, hardware support) which are in line with the 
PTP standard. 
For the simplicity and clarity sake, this section explains all the mechanism of WRPTP without 
classifying them into PTP-profile, PTP-extension, implementation-specific. The distinction is 
made in the last two subchapters where \textit{White Rabbit PTP profile} is defined and 
extensions to PTP listed.

\subsection{Overview}
\label{sec:wrptpOverview}


WRPTP introduces \textit{the White Rabbit Link Setup} (WR Link Setup), which is a process for 
establishing the WR link  (Figure~\ref{fig:wrptpMSGs}). It includes syntonization of the local clock
 over the physical layer, measurement of fixed delays (calibration)  and distribution of the 
information about fixed delays over the link. The WRPTP takes advantage of the Link Delay 
Model to obtain an accurate delay estimation, e.g. it uses the delay asymmetry equation 
\eqref{eq:aqasymm}  for Gigabit Ethernet over Fiber Optic. The additional communication between 
a master and a slave needed to exchange parameters and state information is done through extended 
PTP messaging facilities. 

WRPTP is compliant with standard PTP. Figure~\ref{fig:hybrid-network} depicts the tree topology 
of a \emph{hybrid} WR/IEEE1588 network with grandmaster as a root. The optimal synchronization 
is achieved by connecting WR-nodes directly to the grandmaster and non-WR nodes to WR nodes. 


\begin{figure}[ht!]
  \centering
%  \vspace{-1.3cm}
  \includegraphics[width=0.80\textwidth]{fig/hybrid.ps}
  \caption{Hybrid WR/IEEE1588 network. White Rabbit nodes work
    transparently with PTP nodes. WR ordinary clock 3 is more accurately
      synchronised to the grandmaster than WR ordinary clock 2, which
      is below a PTP boundary clock.}
  \label{fig:hybrid-network}
\end{figure}


\begin{figure}[ht!]
  \centering
%  \vspace{-1.3cm}
  \includegraphics[width=0.80\textwidth]{fig/wrptpMSGs.ps}
  \caption{Simplified overview of the message flow in WRPTP.}
  \label{fig:wrptpMSGs}
\end{figure}


The flow of events for standard PTP which is presented in section~\ref{sec:ptp} is extended 
as depicted in Figure~\ref{fig:wrptpMSGs} and described below (simplified overview):
\newpage
\begin{enumerate}
\item The WR Master periodically sends WR Announce messages with a custom suffix.
\item The WR Slave receives several Announce message(s), recognizes the WR Announce message and uses
the modified BMC algorithm to establish its place in the WR network hierarchy.
\item The WR Slave starts the WR Link Setup by sending the SLAVE\_PRESENT WR Management message.
\item The WR Master sends the LOCK WR Management message to request the WR Slave to start 
syntonization.
\item The WR Slave sends the LOCKED WR Management message as soon as the syntonization process is 
finished (notification from the hardware).
\item The WR Master sends the CALIBRATE WR Management message to request calibration of its 
reception fixed delay.
\item The WR Master sends the CALIBRATED WR Management message as soon as the calibration is 
finished (notification from hardware).
\item The WR Slave sends the CALIBRATE WR Management message to request calibration of its 
reception fixed delay.
\item The WR Slave sends the CALIBRATED WR Management message as soon as the calibration is 
finished (notification from hardware).
\item The WR Master sends the WR\_MODE\_ON WR Management message to indicate completion of  the WR
Link Setup process.
\item The WR Master periodically sends a Sync message (timestamped on transmission, $t_1$)  followed
by a Follow\_UP message which carries $t_1$.
\item The WR Slave receives the Sync message sent by the master (timestamped on reception, $t_2$).
\item The WR Slave receives the Follow\_Up message sent by the master.
\item The WR Slave sends a Delay\_Req message (timestamped on transmission, $t_3$).
\item The WR Master receives the Delay\_Req message sent by the master (timestamped on reception, $t_4$).
\item The WR Master sends a Delay\_Resp message which carries $t_4$.
\item The WR Slave receives the Delay\_Resp.
\item The WR Slave adjusts its clock using the offset and the delay calculated with the timestamps 
($t_{1}$, $t_{2}$, $t_{3}$, $t_{4}$) corrected due to the precise knowledge of the link delay. 
It results in the Slave's synchronization with the Master clock with sub-ns accuracy.
\item Repeat 1, 11-18.
\end{enumerate}

\subsection{Definitions}


The following definitions used in this document are introduced in Clause 3.1 of PTP:\\
\textbf{node}:  A device that can issue or receive Precision Time Protocol (PTP) communications on a
network.\\
\textbf{boundary clock}: A clock that has multiple Precision Time Protocol (PTP) ports in a domain 
and maintains the timescale used in the domain. It may serve as the source of time, i.e., be a 
master clock, and may synchronize to another clock, i.e., be a slave clock.\\
\textbf{ordinary clock}: A clock that has a single Precision Time Protocol (PTP) port in a domain 
and maintains the timescale used in the domain. It may serve as a source of time, i.e., be a master 
clock, or may synchronize to another clock, i.e., be a slave clock.\\
\\
This document introduces the following definitions:\\
\textbf{White Rabbit Master (WR Master)}: a WRPTP-enabled port of a boundary clock or an 
ordinary clock which acts as a source of timing information for another White Rabbit enabled port of 
a boundary clock or an ordinary.\\
\textbf{White Rabbit Slave (WR Slave)}: a WRPTP-enabled port of a boundary clock or an 
ordinary clock in which retrieves timing information send over a link by WR Master\\
\textbf{White Rabbit Switch (WR Switch)}: a boundary clock implementing WRPTP, compatible with 
standard PTP. \\
\textbf{White Rabbit Node (WR Node)}: an ordinary clock implementing WRPTP.






\subsection{WRPTP Data Sets Fields}
\label{sec:wrDS}

The PTP standard defines data sets (DS) to store the static and dynamic variables needed for  the
operation of the protocol (section 8, PTP). WRPTP:
\begin{itemize}
  \item adds fields to DSs defined in PTP standard to store the WR-specific parameters,
  \item defines a new DS: backupParentDS. 
\end{itemize}


\subsubsection{New fields of PTP Data Sets}
Table~\ref{tab:wrDS} defines the additional DS fields required by WRPTP that are not part of
the PTP standard, a description follows.


\begin{table}[ph!]
\caption{WRPTP Data Sets fields}
\centering
\begin{tabular}{| c     |     c  |    p{2.5cm}         | c |    }        \hline   
   
\textbf{DS member}     	& \textbf{DS name}&\textbf{Values}& \textbf{D}ynamic or \textbf{S}tatic 
           \\  
			&              &                         &                                
                                                                                         \\ \hline
wrConfig   		& portDS       & NON\_WR,
					 WR\_S\_ONLY,  
					 WR\_M\_ONLY 
					 WR\_M\_AND\_S		&S \\ \hline
wrMode			& portDS       & NON\_WR,
					 WR\_SLAVE,  
					 WR\_MASTER             &D  \\ \hline
wrModeON 		& portDS       & TRUE, FALSE            &D \\ \hline
wrPortState 		& portDS       & Table~\ref{tab:wrFSMdesc} &D \\ \hline
calibrated 		& portDS       & TRUE, FALSE            &D \\ \hline
deltaTx       		& portDS       & 64 bit value           &D \\ \hline
deltaRx     		& portDS       & 64 bit value           &D \\ \hline
calPeriod   		& portDS       & 32 bit value           &S  \\ \hline
calPattern   		& portDS       & 32 bit value           &S \\ \hline
calPatternLen  		& portDS       & 16 bit value           &S \\ \hline
wrAlpha			& portDS       & 32 bit value           &S   \\ \hline \hline
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
parentWrConfig 		& portDS    	& NON\_WR,
					 WR\_S\_ONLY,  
					 WR\_M\_ONLY 
					 WR\_M\_AND\_S          &D \\ \hline
parentWrMode		& portDS	&  NON\_WR,
					 WR\_SLAVE,  
					 WR\_MASTER            &D  \\ \hline
parentWrModeON 		& portDS       & TRUE, FALSE            &D \\ \hline
parentDeltaTx      	& portDS	& 64 bit value           &D \\ \hline
parentDeltaRx     	& portDS	& 64 bit value           &D\\ \hline
parentCalPeriod   	& portDS       & 32 bit value           &S  \\ \hline
parentCalPattern   	& portDS       & 32 bit value           &S \\ \hline
parentCalPatternLen  	& portDS       & 16 bit value           &S \\ \hline \hline 
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
primarySlavePortNumber	& currentDS	& 16 bits value         &D   \\ \hline

\end{tabular}
\label{tab:wrDS}
\end{table}


\paragraph{wrConfig} Determines predefined function of a WR port.

\paragraph{wrMode} Determines current WR-role of a WR port which is determined by BMC. 

\paragraph{wrModeON} If TRUE, it indicates that WR Link Setup has been performed successfully 
and WR Port is synchronized using WRPTP. TRUE value validates the role defined in wrMode as active.

\paragraph{wrPortState} Stores current state of WRPTP state machine (see chapter~\ref{wrFSM}).

\paragraph{calibrated} If true, it indicates that fixed delays of the given port are known.

\paragraph{deltaTx} Port's $\Delta_{tx}$ measured in picoseconds and multiplied by ${2^{16}}$.

\paragraph{deltaRx} Port's $\Delta_{rx}$ measured in picoseconds and multiplied by ${2^{16}}$.

\paragraph{calPeriod} Calibration period in microseconds. 

\paragraph{calPattern} Medium specific calibration pattern.

\paragraph{calPatternLen} Number of bits of calPattern to be repeated.

\paragraph{wrAlpha} \textit{Relative delay coefficient} described in
 section~\ref{sec:singlefiber}.

\paragraph{primarySlavePortNumber} If 0, no primary Slave is selected. 1, 2 ,...N --value of the
portNumber (clause 7.5.2.3 PTP) selected as the primary Slave, see chapter~\ref{sec:wrBMC}.

\paragraph{parentWrConfig} Stores the value of wrConfig parameter of the parent port.

\paragraph{parentWrMode} Stores the value of wrMode of the parent port.

\paragraph{parentWrModeON} Stores the value of wrModeON of the parent port.

\paragraph{parentCalibrated} Stores the value of calibrated of the parent port.

\paragraph{parentDeltaTx} Stores the value of deltaTx of the parent port.

\paragraph{parentDeltaRx} Stores the value of deltaRx of the parent port.

\paragraph{parentCalPeriod} Stores the value of calPeriod parameter of the parent port. 

\paragraph{parentCalPattern} Stores the value of calPattern parameter of the parent port.

\paragraph{parentCalPatternLen} Stores the value of calPatternLen parameter of the parent port.


\subsubsection{backupParentDS data set specifications}

A boundary clock shall maintain an implementation-specific backupParentDS data set for the purpose 
of qualifying redundant sources of synchronisation, i.e.:
\begin{itemize}
  \item backup paths to the current grandmaster clock, or
  \item paths to alternate grandmaster clock(s).
\end{itemize}
Each entry of the data set contains:

\begin{itemize}
  \item backupParentDS.secondarySlavePortNumber - the number of the port on which the Announce 
	message from backup Parent has been received,
  \item backupParentDS.backupParentSourcePortIdentity - the portIdentity of the port on the backup 
        master.
\end{itemize}

The implementation-specific backupParentDS set shall have a minimum capacity of three backup parent 
records. The order of the records is established using Data Set Comparison Algorithm on $E_{rbest}$ 
which are qualified as backup Masters by modified State Decision Algorithm, record number 0 being the
best (described in the chapter~\ref{StadeDecisionAlg}).



\subsection{Modified Best Master Clock Algorithm}
\label{sec:wrBMC}

\subsubsection{Overview}
The Best Master Clock (BMC) algorithm is used in PTP to compare clocks (determine which 
clock is the \textit{best}) and to recommend the next state of the PTP state machine (Clause 9.3 
of PTP). The BMC is used to prune the physical network topology to obtain logical tree(s) with
\textit{best} clock, grandmaster (preferably the one synchronized to a primary reference time 
source), as a root. In the case when more then one clock in a network is synchronized to a primary 
reference time source, the BMC produces disjoint logic trees. Each tree has a signel grandmaster. 

As a result of BMC, a no more then one port of a Boundary Clock can be in the PTP\_SLAVE state. Such a
solution allows for redundancy of the time source and topology but is not optimal for continuity 
of the synchronization. In example, in the case of a failure of one of the \textit{best} clocks
(reference-synchronized grandmasters), all the slave nodes in its logic tree need to switch 
synchronization to the alternate grandmaster. This requires restart of the estimation of the 
clock draft, mean path delay and offset which might cause fluctuation of time notion. 

The modified BMC allows for more then one \textit{best} clock in a single domain, enabling to create
logic topology with multiple roots. A Boundary Clock running modified BMC can have more then one 
port in the PTP\_SLAVE state. This means that timing information is exchanged between Boundary Clock 
and more then one source of time (Ordinary Clock or Boundary Clock). At any time any of these 
information can be used to perform synchronization, including weighted average from all SLAVE ports 
as mentioned in \cite{Takahide}.

The Best Master Clock algorithm comprises of State Decision Algorithm (SDA), Data Set Comparison
Algorithm (DCA) and defines which fields of Data Sets should be updated depending on the outcome of
SDA. The modifications to BMC are detailed below. DCA is not modified, therefore it is not
mentioned.

\subsubsection{State Decision Algorithm}
\label{StadeDecisionAlg}

The original SDA is depicted in Figure~26 of the PTP standard. The modified SDA, depicted in
Figure~\ref{fig:modifiedSDA}, enforces SLAVE state instead of PASSIVE state. A port being in a
$PTP\_SLAVE$ state as a result of SDA modification (block-8 and block-13 of 
Figure~\ref{fig:modifiedSDA}) is considered secondary SLAVE port. The port which enters 
$PTP\_SLAVE$ state based on the decision at block-11, is considered Primary SLAVE port. 
It means that a Secondary SLAVE port (if present) is connected to the grandmaster clock equally good
or worse by path compared to the one connected to Primary SLAVE port. 

The best qualified Announce messages ($E_{rbest}$, see Clause 9.3.2.3 of PTP) from all Secondary 
SLAVE ports shall be compared with DCA to determine the "second best master" and the lower order
masters. The sequence of events is as follows (the idea originated from \cite{Takahide}):
\begin{itemize}
  \item The best master is computed with the BMC -- the Primary SLAVE port established (block-11 of
	Figure~\ref{fig:modifiedSDA}) and parentDS data set updated (S1), the port's number is 
	stored in currentDS.primarySlavePortNumber.
  \item If the recommended state is $BMC\_SLAVE$ at block-8 or block-13, the master is considered
        a second best master and its data is stored in $E_{srbest}$.
  \item The BMC algorithm is executed on all the $E_{srbest}$.
  \item The best master selected is considered second best master.
  \item The BMC is repeated excluding $E_{srbest}$ of the second best masters.
  \item repeat till the set of $E_{srbest}$ is empty.
\end{itemize}
The order of second and lower level best masters determined in this way is used to update 
backupParentDS data set.


\begin{figure}[ht!]
  \centering
  \includegraphics[width=0.85\textwidth]{fig/SDA.ps}
  \caption{Modified State Decision Algorithm (modifications in red).}
  \label{fig:modifiedSDA}
\end{figure}

\newpage

\subsubsection{Update of Data Sets}

The modifications to the rules governing update of data sets are twofold:
\begin{itemize}
  \item Modifications to already existing "update tables" (Table~13 and Table~16, clause 9.3 of
	PTP) to accommodate new DS fields.
  \item Addition of new "update table" to accommodate new backupParentDS data set and 
	modifications in SDA.
\end{itemize}

\begin{table}[ht!]
\caption{Modification to Table~13 of IEEE1588-2008: Update for state decision code M1 and M2.}
\centering
\begin{tabular}{| c | c |}          \hline
\textbf{Update this field}  		&  \textbf{From the indicated source}  	\\ 
					&  						\\ \hline
currentDS		    		&						\\ \hline
currentDS.primarySlavePortNumber   	& set to 0					\\ \hline

\end{tabular}
\label{tab:modifiedM1M2}
\end{table}

\begin{table}[ht!]
\caption{Modification to Table~16 of IEEE1588-2008: Update for state decision code S1.}
\centering
\begin{tabular}{| c | c |}          \hline
\textbf{Update this field}  		&  \textbf{From the indicated source}  	\\ 
					&  						\\ \hline
currentDS		    		&						\\ \hline
currentDS.primarySlavePortNumber   	& portDS.portIdentity.portNumber		\\ \hline

\end{tabular}
\label{tab:modifiedS1}
\end{table}

\begin{table}[ht!]
\caption{Update for state decision code S2}
\centering
\begin{tabular}{| c | c |}          \hline
\textbf{Update this field}  			&  \textbf{From the indicated source}  	\\ 
						&  					\\ \hline
backupParentDS		    			&					\\ \hline
backupParentDS.secondarySlavePortNumber 	& portDS.portIdentity.portNumber	\\ \hline
backupParentDS.backupParentSourcePortIdentity 	& sourcePortIdentity of $E_{rbest}$	\\ \hline

\end{tabular}
\label{tab:modifiedS2}
\end{table}

\newpage

\subsection{WRPTP Messages}
\label{sec:wrMSG}

White Rabbit benefits from PTP's messaging facilities. It defines \textit{WR Type-Length-Value} 
(WR TLV) to exchange WR-specific information and uses two-step clock delay request-response 
mechanism for synchronization. In particular, it adds a suffix to the Announce message to enable 
recognition of WR nodes and uses Signaling Messages to exchange WR-specific information 
(Figure~\ref{fig:wrptpMSGs}).

A WR Master announces its presence by adding a suffix to the Announce message. 
The suffix is defined by the PTP standard as a set of TLV entities (section 13.4, PTP). 
Unrecognised TLVs are ignored by standard PTP nodes (section 14.1, PTP), but read and interpreted by
White Rabbit nodes. 
The information provided in the WRPTP Announce message is sufficient for another WR port to decide 
whether the WR link can be established and maintained. The WR port receiving 
WRPTP Announce message enters (if conditions are fulfilled) WR Slave mode and starts Link Setup process (performed in the 
$PTP\_UNCALIBRATED$  state, see Appendix~\ref{ap:ptpAndWrFSMS}) requesting the WR Master to do 
the same. During the WR Link Setup, communication between the WR Master and the WR Slave is 
performed using the PTP Signaling Messages carrying WR TLVs. Once the WR link has been established, 
the WR nodes use a PTP delay request-response mechanism (section 11.3, PTP).


\subsubsection{WR Type-Length-Value}
\label{sec:wrTLVtype}

All PTP messages can be extended by means of a standard \textit{type, length, value} (TLV) 
extension mechanism. White Rabbit uses \textit{ORGANIZATION\_EXTENSION} TLV type (tlvType=0x0003) 
which is defined in clause 14.3 of PTP. TLVs represented by this type are designated for vendors 
and standard organizations to extend the protocol for specific needs. The organization specific 
TLV fields for WR extension are defined and described in Table~\ref{tab:wrTLVtype}. 

\begin{table}[h!]
\caption{Organization specific TLV fields for WR (see Table~35, PTP).}
\centering
\begin{tabular}{| c | c | c | c | c | c | c | c | c | c |c|  p{4.3cm}  |}
\hline
\multicolumn{8}{|c|}{\textbf{Bits}} & \textbf{Octets} & \textbf{TLV} & \textbf{WR TLV} & \\
%\hline
\cline{0-7}
7 & 6 & 5 & 4 & 3 & 2 & 1 & 0 &  &  \textbf{Offset}  & \textbf{Content} & \textbf{Description} \\
\hline
\multicolumn{8}{|c|}{tlvType            } & 2 & 0  & 0x0003   & Organization extension, 
								see Table~34 PTP. \\ \hline
\multicolumn{8}{|c|}{lengthField        } & 2 & 2  & 8+N      & N is an even number of wrDataField
								octets. \\ \hline
\multicolumn{8}{|c|}{OrganizatinId      } & 3 & 4  & 0x080030 & OUI owned by CERN. \\
\hline
\multicolumn{8}{|c|}{organizationSubType} & 2 & 7  & magicNumber & e.g.: 0xDEAD - identifies WRPTP 
						      within protocols identified by CERN's OUI. \\
\cline{9-12} %\hline
\multicolumn{8}{|c|}{}                    & 1 & 9  & versionNumber& 0x01 - WRPTP version.\\
 \hline
\multicolumn{8}{|c|}{DataField}           & 2 & 10 & wrMessageID  & WR-specific messages identifier,
								    see Table~\ref{tab:wrMessageId}. \\
\cline{9-12} %\hline 
\multicolumn{8}{|c|}{  }                 & N & 12 & wrDataField   & Content of WR-specific message. \\
\hline
\end{tabular}
\label{tab:wrTLVtype}
\end{table}
The WR-specific TLVs are identified by wrMessageID which is defined in 
Table~\ref{tab:wrMessageId}, the different types of WR messages are described in subsequent 
chapters.

\begin{table}[ht!]
\caption{White Rabbit Message ID values}
\centering
\begin{tabular}{| l | p{2.5cm} | c | c |}          \hline
\textbf{managementId name}  &  \textbf{wrMessageId value (hex)} &  \textbf{Sent in message type} \\ 
& &  \\ \hline
SLAVE\_PRESENT     &  0x1000 & Signaling \\ \hline
LOCK               &  0x1001 & Signaling \\ \hline
LOCKED             &  0x1002 & Signaling \\ \hline
CALIBRATE          &  0x1003 & Signaling \\ \hline
CALIBRATED         &  0x1004 & Signaling \\ \hline
WR\_MODE\_ON       &  0x1005 & Signaling \\ \hline
ANN\_SUFIX	   &  0x2000 & Announce  \\ \hline
\end{tabular}
\label{tab:wrMessageId}
\end{table}

\subsubsection{WRPTP Announce Message}
\label{sec:wrAnnounceMSG}


The standard PTP Announce Message is suffixed by one entity of WR TLV. 
The WRPTP Announce message has the structure defined in Table~\ref{tab:wrAnnounce}. The $dataField$
of the suffix WR TLV stores the $wrFlags$ which are defined in Table~\ref{tab:wrFlags}.

\begin{table}[h!]
\caption{White Rabbit Announce Message}
\centering
\begin{tabular}{| c | c | c | c | c | c | c | c | c | c | p{4.5cm} |}
\hline
\multicolumn{8}{|c|}{\textbf{Bits}} & \textbf{Octets} & \textbf{TLV}  & \textbf{Content} \\
%\hline
\cline{0-7}
7 & 6 & 5 & 4 & 3 & 2 & 1 & 0 &     & \textbf{Offset}  &  \\
\hline
\multicolumn{8}{|c|}{header        }  & 34 & 0  & section 13.3, PTP. \\ \hline
\multicolumn{8}{|c|}{body          }  & 30 & 34 & section 13.5, PTP.         \\ \hline
\multicolumn{8}{|c|}{tlvType       }  & 2  & 64 & 0x0003, see \ref{sec:wrTLVtype}. \\ \hline
\multicolumn{8}{|c|}{lengthField   }  & 2  & 66 & 0xA.   \\ \hline
\multicolumn{8}{|c|}{OrganizationId}  & 3  & 68 & 0x080030.               \\ \hline
\multicolumn{8}{|c|}{magicNumber   }  & 2  & 71 & 0xDEAD.               \\ \hline
\multicolumn{8}{|c|}{versionNumber }  & 1  & 73 & 0x01.               \\ \hline
\multicolumn{8}{|c|}{wrMessageId   }  & 2  & 74 & 0x0010.               \\ \hline
\multicolumn{8}{|c|}{wrFlags       }  & 2  & 76 & see Table~\ref{tab:wrFlags}.    \\ \hline
\end{tabular}
\label{tab:wrAnnounce}
\end{table}


\begin{table}[ht!]
\caption{White Rabbit flags (unused flags are reserved and shall be written to 0, ignored when read)}
\centering
\begin{tabular}{| c | c | c | c | p{5.3cm} |}          \hline

\textbf{Octet} &\textbf{Bit} & \textbf{Message type} & \textbf{Name}  & \textbf{Description}  \\ 
               &             &                       &                &              \\ \hline
0              &0   	& \multirow{2}{*}{Announce} 	& \multirow{2}{*}{wrConfig}  &
\multirow{2}{5.3cm}{The value of wrConfig parameter from portDS.} \\ \cline{1-2}
0              &1            &				& 		&\\ \hline
0              &2            & Announce              & calibrated     & TRUE if the source port 
									is calibrated.\\
									\hline
0              &3            & Announce              & wrModeON     & The value of wrModeON
									parameter of portDS.\\
									  \hline
\end{tabular}
\label{tab:wrFlags}
\end{table}

\newpage


\subsubsection{WRPTP Signaling Messages}
\label{wrSignalingMSG}

White Rabbit uses Signaling Messages (defined in clause 13.12 of PTP) to exchange WR-specific 
information, except for the flags transported in the suffix to Announce message. 

The Signaling Messages conforms to the format presented in Table~\ref{tab:wrSignalingMSG}. Each WR
Signaling Message transports single WR TLV structured as defined in Section~\ref{sec:wrTLVtype}.
The targetPortIdentity shall be always set to clockIdentity of the port on the other side of the 
link (conveyed in the Announce Message). 

Signaling Message are used in WRPTP to trigger transitions in WR state machines and exchange 
WR-specific parameters. They are distinguished by the \textit{wrMessageId} field of \textit{WR TLV}, 
defined in Table~\ref{tab:wrMessageId}. Signaling Messages are exchanged only within single 
link connection (no forwarding) which is ensured by setting targetPortId appropriately. 
The rest of this subsection describes the WRPTP Signaling Messages in details.

\begin{table}[h!]
\caption{PTP Signaling message fields (Table 33, PTP) }
\centering
\begin{tabular}{| c | c | c | c | c | c | c | c | c | c |}
\hline
\multicolumn{8}{|c|}{\textbf{Bits}}               & \textbf{Octets} & \textbf{TLV}  \\
%\hline
\cline{0-7}
7 & 6 & 5 & 4 & 3 & 2 & 1 & 0                     &                 & \textbf{Offset}    \\
\hline
\multicolumn{8}{|c|}{header                    }  & 34             & 0             \\ \hline
\multicolumn{8}{|c|}{targetPortIdentity        }  & 10             & 34            \\ \hline
\multicolumn{8}{|c|}{WR TLV                    }  & M              & 44             \\ \hline
\end{tabular}
\label{tab:wrSignalingMSG}
\end{table}


\paragraph{SLAVE\_PRESENT} Message sent by the WR Slave to the WR Master.  It initiates the WR Link
Setup process in the WR Master. The message shall have the form specified in 
Table~\ref{tab:wrOtherTLV}.

\paragraph{LOCK} Message sent by the WR Master to the WR Slave to request the start  of frequency
locking. The message shall have the form specified in Table~\ref{tab:wrOtherTLV}.

\paragraph{LOCKED} Message sent by the WR Slave to the WR Master. It indicates successful 
completion of frequency locking. The message shall have the format specified in 
Table~\ref{tab:wrOtherTLV}.


\begin{table}[h!]
\caption{WR TLV for SLAVE\_PRESENT, LOCK, LOCKED AND WR\_MODE\_ON Signaling Messages.}
\centering
\begin{tabular}{| c | c | c | c | c | c | c | c | c | c | p{4.5cm} |}
\hline
\multicolumn{8}{|c|}{\textbf{Bits}} & \textbf{Octets} & \textbf{TLV}  & \textbf{Content} \\
%\hline
\cline{0-7}
7 & 6 & 5 & 4 & 3 & 2 & 1 & 0 &     & \textbf{Offset}  &  \\
\hline
\multicolumn{8}{|c|}{tlvType       }  & 2  & 0  & 0x0003, see \ref{sec:wrTLVtype}. \\ \hline
\multicolumn{8}{|c|}{lengthField   }  & 2  & 2  & 0x8.   \\ \hline
\multicolumn{8}{|c|}{OrganizationId}  & 3  & 4  & 0x080030.               \\ \hline
\multicolumn{8}{|c|}{magicNumber   }  & 2  & 7  & 0xDEAD.               \\ \hline
\multicolumn{8}{|c|}{versionNumber }  & 1  & 9  & 0x01.               \\ \hline
\multicolumn{8}{|c|}{wrMessageId   }  & 2  & 10 & Defined in Table~\ref{tab:wrMessageId}.\\ \hline
\end{tabular}
\label{tab:wrOtherTLV}
\end{table}



\paragraph{CALIBRATE} Message sent by the WR node entering the REQ\_CALIBRATION state  (see
section~\ref{wrFSM}). It informs the other node 
whether sending a calibration pattern (see section~\ref{sec:fixedDelays}) is required  (defined by
the value of \textit{calibrationSendPattern} flag). 
If calibration is required, it carries a set of parameters describing 
the calibration pattern to be sent. The message format and parameters are described  in
Table~\ref{tab:wrCalibrateTLV}.


\begin{table}[h!]
\caption{WR TLV CALIBRATE Signaling Message }
\centering
\begin{tabular}{| c | c | c | c | c | c | c | c | c | c | p{7.5cm} |}
\hline
\multicolumn{8}{|c|}{\textbf{Bits}} & \textbf{Octets} & \textbf{TLV}  & \textbf{Content} \\
%\hline
\cline{0-7}
7 & 6 & 5 & 4 & 3 & 2 & 1 & 0 &     & \textbf{Offset}  &  \\
\hline
\multicolumn{8}{|c|}{tlvType               }  & 2  & 0  & 0x0003, see \ref{sec:wrTLVtype}. \\ \hline
\multicolumn{8}{|c|}{lengthField           }  & 2  & 2  & 0x14.   \\ \hline
\multicolumn{8}{|c|}{OrganizationId        }  & 3  & 4  & 0x080030.               \\ \hline
\multicolumn{8}{|c|}{magicNumber           }  & 2  & 7  & 0xDEAD.               \\ \hline
\multicolumn{8}{|c|}{versionNumber         }  & 1  & 9  & 0x01.               \\ \hline
\multicolumn{8}{|c|}{wrMessageId           }  & 2  & 10 & CALIBRATE.               \\ \hline
\multicolumn{8}{|c|}{calibrationSendPattern}  & 2  & 12  & The value determines whether calibration 
						       pattern should be sent. If the value
							is 0x1, the calibration pattern is sent. If
							the value is 0x0, the calibration pattern is
							not sent. \\ \hline
\multicolumn{8}{|c|}{calibrationPeriod}     & 4 & 14  & The  value defines the time (in microseconds)
							for which the calibration pattern should be
							sent by the receiving node.\\ \hline
\multicolumn{8}{|c|}{calibrationPattern}    & 4 & 18 & The  value defines the calibration pattern
							which should be sent by the receiving
							node.\\ \hline
\multicolumn{8}{|c|}{calibrationPatternLen} & 2 & 22 & The  value defines the number of bits of
							\textit{calibrationPattern} field which
							should be used as repeated pattern (starting
							  with the LSB). \\\hline
\end{tabular}
\label{tab:wrCalibrateTLV}
\end{table}


\newpage



\paragraph{CALIBRATED} Message sent by a WR node entering the \textit{CALIBRATED} state. 
If preceded by the \textit{CALIBRATE} message with \textit{calibrationSendPattern} set to TRUE, 
it indicates successful completion of the calibration. The message provides the other node with the 
values of its fixed delays ($\Delta_{tx}$ and $\Delta_{rx}$). 
The message shall have the format specified in Table~\ref{tab:wrCalibratedTLV}.

\begin{table}[ht]
\caption{WR TLV for CALIBRATED Signaling Message.}
\centering
\begin{tabular}{| c | c | c | c | c | c | c | c | c | c |  p{7.5cm}  |}
\hline
\multicolumn{8}{|c|}{\textbf{Bits}} & \textbf{Octets} & \textbf{TLV} &\\
%\hline
\cline{0-7}
7 & 6 & 5 & 4 & 3 & 2 & 1 & 0 &  &  \textbf{Offset}  & \textbf{Content}  \\
\hline
\multicolumn{8}{|c|}{tlvType               }  & 2  & 0  & 0x0003, see \ref{sec:wrTLVtype}. \\ \hline
\multicolumn{8}{|c|}{lengthField           }  & 2  & 2  & 0x28.   \\ \hline
\multicolumn{8}{|c|}{OrganizationId        }  & 3  & 4  & 0x080030.               \\ \hline
\multicolumn{8}{|c|}{magicNumber           }  & 2  & 7  & 0xDEAD.               \\ \hline
\multicolumn{8}{|c|}{versionNumber         }  & 1  & 9  & 0x01.               \\ \hline
\multicolumn{8}{|c|}{wrMessageId           }  & 2  & 10 & CALIBRATED.               \\ \hline
\multicolumn{8}{|c|}{deltaTx               }  & 16 & 12 & The value of $\Delta_{tx_m}$ measured in 	
						   picoseconds and multiplied by ${2^{16}}$.\\
						    \hline
\multicolumn{8}{|c|}{deltaRx               }  & 16 & 28 &  The value of $\Delta_{rx_m}$ measured in
						      picoseconds and multiplied by ${2^{16}}$.\\
							\hline
\end{tabular}
\label{tab:wrCalibratedTLV}
\end{table}




\paragraph{WR\_MODE\_ON} Message sent by WR Master to WR Slave. It indicates successful completion 
of 
the WR Link Setup process and requests the WR~Slave to enter WR~mode. The message shall have the 
format specified in Table~\ref{tab:wrOtherTLV}.

\newpage

\subsection{PTP State Machine}
\label{ptpStateMachine}

The WR Link Setup (WR state machine) is performed in the PTP\_UNCALIBRATED state of PTP state 
machine (see Appendix~\ref{ap:ptpAndWrFSMS}). Therefore it is essential for a WR port to implement 
this state as specified in the PTP state machine (\textcolor{blue}{2} in 
Figure~\ref{fig:modifiedPtpFSM}): a transition state 
between the  LISTENING or PRE\_MASTER or MASTER or PASSIVE state and the SLAVE state.


WRPTP specifies implementation-specific SYNCHRONIZATION\_FAULT PTP state transition event (defined in
Clause 9.2.6.12). It shell be implemented as evaluation of two WR-specific parameters: $wrModeON$ 
and $parentWrModeON$ by the port being in WR\_SLAVE mode and PTP\_SLAVE state. If the value of at least
on of the parameters is FALSE, the SYNCHRONIZATION\_FAULT even shell occur and PTP\_UNCALIBRATED 
entered. Consequently the WR Link Setup is performed. Such implementation enables forced 
re-calibration of WR nodes which can be triggered by both, WR Master and WR Slave.


WRPTP introduces new state transition event to the original PTP state machine: 
\textit{CALIBRATION\_FORCED\_BY\_SLAVE}. It is depicted in red (number 2)
 in Figure~\ref{fig:modifiedPtpFSM}. The event shall be instantiated by the reception of
 $SLAVE\_PRESENT$ Signaling Message on WR-Master-enabled port (the value of portDS.wrConfig's 
 is WR\_M\_AND\_S or WR\_M\_ONLY) in $PTP_MASTER$ state. Such state transition enables WR Slave
 to initiate WR Link Setup on WR  Master.


\begin{figure}[ht!]
  \centering
  \includegraphics[width=0.85\textwidth]{fig/modifiedPtpFSM.ps}
  \caption{Modified PTP Finite State Machine.}
  \label{fig:modifiedPtpFSM}
\end{figure}



\newpage

\subsection{White Rabbit State Machine}
\label{wrFSM}

The White Rabbit finite state machine (WR FSM) controls the process of establishing a White Rabbit 
link between a WR Master and a WR Slave (WR Link Setup). It involves recognition of two compatible 
WR nodes, syntonization over the physical layer, measurement of fixed delays and exchange of 
their values across the link. The procedure differs between WR Master and WR Slave, therefore 
three states are Slave-only (entered only if the node is in WR Slave mode) and one state is 
Master-only (entered only if the node is in WR Master mode). The WR FSM shall be 
executed in the PTP UNCALIBRATED state, it is depicted in Figure~\ref{fig:wrFSM} and described  in
the rest of 
this section.

A typical flow of a complete WR Signalin Message exchange between a WR Slave and a WR Master 
during WR Link Setup is presented in Appendix~\ref{ap:wr_lsu_flow}. For clarity, the cooperation of 
PTP and WR state machines from the power up is presented in Appendix~\ref{ap:ptpAndWrFSMS}.

\subsubsection{Condition to start the WR FSM by WR Slave}
\label{sec:wrSlaveFSMstart}

The WR FSM in a WR Slave-enabled port (portDS.wrConfig equals \textit{WR\_M\_ONLY} OR 
\textit{WR\_M\_AND\_S}) exits the IDLE state and starts execution by entering the \textit{PRESENT}
state, only when the PTP state machine enters the PTP UNCALIBRATED state 
as a result of STATE\_DECISION\_EVENT: Recommended State == BMC\_SLAVE (\textcolor{blue}{1} in 
Figure~\ref{fig:modifiedPtpFSM}) and the following conditions
are met:
\begin{itemize}
\item the port is WR Slave-enabled: \\ 
	\textit{portDS.portWrConfig = ($WR\_S\_ONLY$ \textbf{OR} $WR\_M\_AND\_S$)} \textbf{AND}
\item the parent port is WR Master-enabled: \\ 
	\textit{portDS.parentPortWrConfig = ($WR\_M\_ONLY$ \textbf{OR} $WR\_M\_AND\_S$)} \textbf{AND}
\item at least one of the ports on the link is not in WR Mode:  \\ 
	\textit{portDS.portWrMode = FALSE \textbf{OR} portDS.ParentPortWrMode = FALSE}.
\end{itemize}

\subsubsection{Condition to start the WR FSM by WR Master}
\label{sec:wrMasterFSMstart}


The WR FSM in a WR Master-enabled port (\textit{$WR\_M\_ONLY$ \textbf{OR} $WR\_M\_AND\_S$})
exits the IDLE state and starts execution by entering the \textit{M\_LOCK} state only when 
PTP state machine enters the PTP\_UNCALIBRATED state as a result of 
CALIBRATION\_FORCED\_BY\_SLAVE transition event (\textcolor{blue}{2} in 
Figure~\ref{fig:modifiedPtpFSM}).


\newpage


\paragraph{State Description}

Table~\ref{tab:wrFSMdesc} specifies the WR states notation used in Figure~\ref{fig:wrFSM}.

\begin{table}[hp!]
\caption{WR state definition}
\centering
\begin{tabular}{| c | p{11cm} |}          \hline
\textbf{PTP portState}  &  \textbf{Description} \\ 
&   \\ \hline
\small
IDLE               &  The WR FSM shall be in the IDLE state if the PTP FSM is in a state other than 
		      UNCALIBRATED. \\ \hline
PRESENT            &  Slave-only state. The WR Slave sends a SLAVE\_PRESENT message to the WR 
		      Master and waits for the $LOCK$ message.\\ \hline
M\_LOCK            &  Master-only state. The WR master waits for the WR Slave to finish 
		      successfully the locking process. \\ \hline
S\_LOCK            &  Slave-only state. The WR Slave locks its logic to the frequency  distributed
		      over physical layer by the WR Master. \\ \hline
LOCKED             &  Slave-only state. The WR Slave is syntonized, it sends a \textit{LOCKED} 
		      message to the WR Master and waits for the \textit{CALIBRATE} message.  \\
		      \hline
REQ\_CALIBRATION   &  In this state, optional calibration of the node's reception fixed delay  can
		      be performed. 
		      The node sends a \textit{CALIBRATE} message to the other node. If the 
		      calibration is needed,
		      (\textit{calibrated} is set to false), the \textit{calibrationSendPattern} 
		      flag in the \textit{CALIBRATE} message 
		      is sent to TRUE (0x1). If the calibration is not needed, the
		      \textit{calibrationSendPattern} flag is set to FALSE (0x0).  
		      If calibration is not needed, the next state is entered directly, otherwise an
		      indication from the hardware
		      that the calibration has been finished successfully is awaited. \\ \hline
CALIBRATED         &  The node sends a \text{CALIBRATED} message with information about its fixed 
		      delays. \\ \hline
RESP\_CALIB\_REQ   &  The node's action in this state depends on the value of the 
		      \textit{calibrationSendPattern} flag received in the \textit{CALIBRATE}
		      message. A TRUE value of the flag
		      indicates that the calibration pattern shall be enabled. The pattern shall  be
		      disabled after \textit{calibrationPeriod} 
		      or on reception of the \textit{CALIBRATED} message. If the value of  the
		      \textit{calibrationSendPattern} flag is FALSE, 
		      the \textit{CALIBRATED} message is awaited for a default timeout.  On
		      reception of the \textit{CALIBRATED} message, the next state is entered.\\
		      \hline
WR\_LINK\_ON       &  The value of \textit{wrMode} is set to TRUE and the \textit{IDLE}  state is
		      entered. \\ \hline
\end{tabular}
\label{tab:wrFSMdesc}
\end{table}





\begin{figure}[ht!]
  \centering
%  \vspace{-1.3cm}
  \includegraphics[width=1.00\textwidth]{fig/wrFSM.ps}
  \caption{White Rabbit state machine.}
  \label{fig:wrFSM}
\end{figure}






\paragraph{WR FSM Transition Events and Conditions}

\begin{description}
\item[]
\item[POWERUP]				Turning on power to the device or reseting.
\item[WR LINK SETUP REQUIRED DECISION] 	(abrv. D\_WR\_SETUP\_REQ) 	Event indicating that a  WR
      Link Setup is required and that the WR FSM should be executed starting at the
      \textit{PRESENT} state, see section~\ref{sec:wrSlaveFSMstart}.
\item[LOCK MESSAGE] 			(abrv. M\_LOCK) 		WR $LOCK$ Management message
      which triggers frequency locking over the physical layer.
\item[LOCKED HARDWARE EVENT] 		(abrv. HW\_LOCKED) 		Indication from the hardware
      that frequency locking has been completed successfully.
\item[LOCKED MESSAGE] 			(abrv. M\_LOCKED)		WR $LOCKED$ Management 
      message which notifies the WR Master that frequency locking has been completed 
									successfully by the WR Slave.
\item[CALIBRATE MESSAGE] 		(abrv. M\_CALIBRATE) 		WR $CALIBRATE$ Management 
      message. It requests the recipient to enter the
      \textit{RESP\_CALIB\_REQ} state, indicates whether sending of the calibration pattern is
      required and provides calibration parameters.
\item[CALIBRATED HARDWARE EVENT] 	(abrv. HW\_CALIBRATE) 		Notification from the
      hardware indicating that calibration has been completed successfully.
\item[CALIBRATED MESSAGE] 		(abrv. M\_CALIBRATED)		WR  \text{CALIBRATED}
      Management message. It indicates that the node is calibrated. If the \textit{CALIBRATED}
      message is received when the calibration pattern is being sent by the recipient, 
      sending of the pattern shall be disabled. The message carries information about fixed delays
      of the sending node.
\item[WR MODE ON] 			(abrv. M\_WR\_MODE\_ON)		WR \textit{WR\_MODE\_ON}
      Management message. It indicates that the WR Master finished successfully the WR Link Setup
      and set the \text{wrMode} flag to TRUE. It requests the WR Slave to set the \text{wrMode} flag
      to TRUE.
\item[RETRY $n_{name}$]							The White Rabbit state
      machine waits in a given state for a transition event only for a limited time
      (\textit{TIMEOUT}) The states to which this rule applies are the following: \textit{M\_LOCK}, 
      \textit{REQ\_CALIBRATEION}, \textit{CALIBRATED}, \textit{PRESENT}, \textit{S\_LOCK}, 
      \textit{LOCKED},\textit{RESP\_CALIB\_REQ}. After the \textit{TIMEOUT} expires, the state is
      re-entered for \\  $n_{\{M\_LOCK, REQ\_CAL,
      CALIBed, PRESENT, S\_LOCK, LOCKED, RESP\_CALIB\_RESP\}}$ number of times.
\item[EXCEED TIMEOUT RETRIES] 		(abrv. EXC\_TIMEOUT\_RETRY)	Indicates that the state 
      has been re-entered for a set number of times \\($n_{\{M\_LOCK, REQ\_CAL, CALIBed, PRESENT,
      S\_LOCK, LOCKED, RESP\_CALIB\_RESP\}}$) and the \textit{TIMEOUT} has expired for the $n + 1$
      time.
									  
\end{description}

\newpage

\subsection{White Rabbit PTP Profile Summary}
\subsubsection{Identification}


\begin{table}[ht!]
\caption{Profile print form (clause 19.3.3 of PTP)}
\centering
\begin{tabular}{| c | c |}          					\hline
\multicolumn{2}{|c|}{\textbf{PTP Profile}}  				  \\ \hline
prifleName           &  White Rabbit     				  \\ \hline
profileVersion       &  1.0               				  \\ \hline
profileIdentifier    &  08-00-03-00-01-00 				  \\ \hline
organizationName     &  European Organization for Nuclear Research (CERN) \\ \hline
sourceIdentification &  http://www.ohwr.org/projects/white-rabbit         \\ \hline
\end{tabular}
\label{tab:wrPtpProfile}
\end{table}

\subsubsection{PTP attribute values}

All nodes shall support the ranges and shall have the default initialization values for 
the attributes as follows:
\begin{itemize}
  \item portDS.logSyncInterval: The default initialization value shall be 0. The configuration range
	shall be -1 to 6.
  \item defaultDS.priority1: The default initialization value shall be 64.
\end{itemize}

\subsubsection{PTP Optins}

All options of 15.5.4.1.7 and clause 17 of PTP are permitted (??). By default, these options shall be 
inactive unless specifically activated by a management procedure.
\\
Node management shall implement the management message mechanism of IEEE1588-2008 standard.
\\
The best master algorithm shall be the algorithm specified in Chapter~\ref{sec:wrBMC} of this 
document (Modified BMC).
\\
The delay request-response mechanism shall be the default and only path delay measurement mechanism.
\\
The TLV mechanism described in Chapter~\ref{sec:wrMSG} shall be supported.

\subsection{WRPTP Extensions to PTP}

WRPTP extends PTP by adding state transition event defined in subchapter~\ref{ptpStateMachine}.

\subsection{Implementation-specific additions to PTP required by WRPTP}
 
WRPTP shell implement implementation-specific features:
\begin{itemize}
  \item issuing PTP messages with WR TLV (i.e.suffixed Announce and Signaling messages),
	as described in subchapter~\ref{sec:wrAnnounceMSG},
  \item proper handling of PTP message with WR TLV (i.e. Signaling and suffixed Announce messages)
	as described in subchapter~\ref{sec:wrAnnounceMSG},
  \item WR state machine as defined in subchapter~\ref{wrFSM},
  \item WR data set and additional WR-specific fields to PTP data sets as defined in 
	subchapter~\ref{sec:wrDS},
  \item SYNCHRONIZATION\_FAULT state transition as defined in subchapter~\ref{ptpStateMachine}.
\end{itemize}


\newpage
\begin{center}
\huge Appendix
\end{center} 

\normalsize
\appendix
\section{PTP State Machine}
\label{ptpFSM}

 \begin{figure}[ht!]
   \centering
   \includegraphics[width=0.85\textwidth]{fig/ptpFSM.ps}
   \caption{State machine for a full implementation of PTP (Figure 23, IEEE1588).}
   \label{fig:ptpFSM}
 \end{figure}
 
 \newpage



\begin{table}[hp!]
\caption{PTP portState definition (Table 10, IEEE1588).}
\centering
\begin{tabular}{| c | p{9.5cm} |}          \hline
\textbf{PTP portState}  &  \textbf{Description} \\ 
&   \\ \hline
\small
INITIALIZING       &  \small While a port is in the INITIALIZING state, the port initializes its 
		      data sets, hardware, and
		      communication facilities. No port of the clock shall place any PTP messages
		      on its
		      communication path. If one port of a boundary clock is in the INITIALIZING 
		      state, then all ports
		      shall be in the INITIALIZING state.  \\ \hline
FAULTY             &  \small The fault state of the protocol. A port in this state shall not place 
		      any PTP messages except for
		      management messages that are a required response to another management 
		      message on its
		      communication path. In a boundary clock, no activity on a faulty port  
		      shall affect the other ports
		      of the device. If fault activity on a port in this state cannot be confined 
		      to the faulty port, then all
		      ports shall be in the FAULTY state. \\ \hline
DISABLED           &  \small The port shall not place any messages on its communication path. In a
		      boundary clock, no
		      activity at the port shall be allowed to affect the activity at any other
		      port of the boundary clock.
		      A port in this state shall discard all PTP received messages except for 
		      management messages. \\ \hline
LISTENING          &  \small The port is waiting for the announceReceiptTimeout to expire or to
		      receive an Announce
		      message from a master. The purpose of this state is to allow orderly addition 
		      of clocks to a
		      domain. A port in this state shall not place any PTP messages on its 
		      communication path except
		      for Pdelay\_Req, Pdelay\_Resp, Pdelay\_Resp\_Follow\_Up, or signaling 
		      messages, or management
		      messages that are a required response to another management message. \\ \hline
PRE\_MASTER         & \small The port shall behave in all respects as though it were in the 
		      MASTER state except that it shall
		      not place any messages on its communication path except for Pdelay\_Req, 
		      Pdelay\_Resp,
		      Pdelay\_Resp\_Follow\_Up, signaling, or management messages.  \\ \hline
MASTER             &  \small The port is behaving as a master port.   \\ \hline
PASSIVE            &  \small The port shall not place any messages on its communication path except 
		      for Pdelay\_Req,
		      Pdelay\_Resp, Pdelay\_Resp\_Follow\_Up, or signaling messages, or management
		      messages that
		      are a required response to another management message.\\ \hline
UNCALIBRATED       &  \small One or more master ports have been detected in the domain. The 
		      appropriate master port has
		      been selected, and the local port is preparing to synchronize to the selected 
		      master port. This is a
		      transient state to allow initialization of synchronization servos, updating of
		      data sets when a new
		      master port has been selected, and other implementation-specific activity.   
		      \\ \hline
SLAVE              &  \small The port is synchronizing to the selected master port.  \\ \hline
\end{tabular}
\label{tab:ptp_port_state_def}
\end{table}


\newpage


\section{Measurement of fixed delays for Gigabit Ethernet over Optic Fiber}
\label{sec:calibForGigbitE}

The variation of $\Delta_{\{tx_m, rx_s, tx_s, rx_m\}}$ delays is often caused by 
the PHY's serializer / deserializer (SerDes), phase locked loop (PLL) or clock and 
data recovery circuitry (CDR). The delay on the PHY can be measured by detecting 
the phase shift between SerDes I/O and Tx/Rx clock. This can be done, for example, by sending 
a repeated pattern of five "0" and five "1" (0000011111) over Gigabit Ethernet. 
Such signal creates a 125~MHz clock on the SerDes I/O. Since the Tx/Rx clock 
frequency is 125~MHz, the phase shift between the SerDes I/O and the Tx/Rx 
clocks is equal to the fixed delay of the PHY (see Figure~\ref{fig:wrCalibration}). 
The repeated pattern of five "0" and five "1" is an example of 
\textit{calibration pattern} which is defined by the node requesting a calibration.

The calibration pattern used to describe the methods is not compliant with the 8b/10b encoding 
standard. 
For real-life implementation, a 8b/10b compliant pattern is needed.

\begin{figure}[ht!]
  \centering
  \includegraphics[width=0.60\textwidth]{fig/calibrate.ps}
  \caption{Measurement of fixed delays $\Delta_{\{tx, rx\}}$ in Gigabit Ethernet-based WR node with 
not fully-deterministic PHY.}
  \label{fig:wrCalibration}
\end{figure}

Measurement of fixed delays for Gigabit Ethernet over optic fiber, and any other medium, is optional. 
It is not needed if deterministic PHYs or internal FPGA transceivers which can be internally 
characterized \cite{Peek2010} are used. In such case, the information about the fixed delays is 
distributed across the link without preceding measurement.

\newpage

\section{Typical flow of WR Management message exchange}
\label{ap:wr_lsu_flow}
\begin{figure}[ht!]
  \centering
%  \vspace{-1.3cm}
  \includegraphics[width=1.0\textwidth]{fig/wrMSGsExchangeFlow.ps}
  \caption{Typical flow of events (no exceptions) during WR Link Setup.}
  \label{fig:wrFSMcommun}
\end{figure}

\newpage


\section{PTP and WR FSMs from POWER ON use case}
\label{ap:ptpAndWrFSMS}
\begin{figure}[ht!]
  \centering
%  \vspace{-1.3cm}
  \includegraphics[width=1.0\textwidth]{fig/ptpAndWrFSMs.ps}
  \caption{PTP and WR FSMs from POWER ON use case}
  \label{fig:wrFSMcommun}
\end{figure}

\newpage

\begin{thebibliography}{9}



\bibitem{IEEE1588}
  IEEE Std 1588-2008
  \emph{IEEE Standard for a Precision Clock Synchronization Protocol for Networked Measurement and 
Control Systems}.
  IEEE Instrumentation and Measurement Society, New York,
  2008,
  http://ieee1588.nist.gov/.

\bibitem{Peek2010}
  P.P.M. Jansweijer, H.Z. Peek,
  \emph{Measuring propagation delay over a 1.25 Gbps bidirectional data link}.
  National Institute for Subatomic Physics, Amsterdam,
  2010,\\
  http://www.nikhef.nl/pub/services/biblio/technicalreports/ETR2010-01.pdf.

\bibitem{Takahide}
  Takahide Murakami and Yukio Horiuchi,
  \emph{A Master Redundancy Technique in IEEE 1588 Synchronization with a Link Congestion
  Estimation}.
  ISPCS Proccedings,
  2010,\\


\end{thebibliography}


\end{document}
